nk20-scripts/management/commands/import_transaction.py

# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
# SPDX-License-Identifier: GPL-3.0-or-later

import re
import psycopg2 as pg
import psycopg2.extras as pge
import pytz
import datetime
import copy

from django.contrib.auth.models import User
from django.utils.timezone import make_aware
from django.db import transaction
from django.contrib.contenttypes.models import ContentType

from note.models import (TemplateCategory,
                         TransactionTemplate,
                         Transaction,
                         RecurrentTransaction,
                         SpecialTransaction,
                         MembershipTransaction,
                         )
from note.models import Note, NoteClub
from activity.models import Guest, GuestTransaction, Entry

from member.models import Membership
from treasury.models import Remittance, SpecialTransactionProxy, SogeCredit
from ._import_utils import ImportCommand, BulkCreateManager, timed

MAP_TRANSACTION = dict()
MAP_REMITTANCE = dict()

# from member/fixtures/initial
BDE_PK = 1
KFET_PK = 2

# from note/fixtures/initial
NOTE_SPECIAL_CODE = {
    "espèce": 1,
    "carte": 2,
    "chèque": 3,
    "virement": 4,
}
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2

CT = {
    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}

def get_date_end(date_start):
    date_end = copy.deepcopy(date_start)
    if date_start.month >= 8:
        date_end = date_start.replace(year=date_start.year + 1)
    date_end = date_end.replace(month=9, day=30)
    return date_end


class Command(ImportCommand):
    """
    Import command for People base data (Comptes, and Aliases)
    """

    def add_arguments(self, parser):
        parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
        parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
        parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
                                                                                       "don't save them")

    @timed
    def import_buttons(self, cur, chunk_size, import_buttons):
        self.categories = dict()
        self.buttons = dict()
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute("SELECT * FROM boutons;")
        n = cur.rowcount
        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["label"])
            if row["categorie"] not in self.categories:
                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
                cat.save()
                self.categories[row["categorie"]] = cat.pk
            obj_dict = {
                "pk": row["id"],
                "name": row["label"],
                "amount": row["montant"],
                "destination_id": self.MAP_IDBDE[row["destinataire"]],
                "category_id": self.categories[row["categorie"]],
                "display": row["affiche"],
                "description": row["description"],
            }
            if row["label"] in self.buttons:
                obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
            if import_buttons:
                bulk_mgr.add(TransactionTemplate(**obj_dict))
            self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
        bulk_mgr.done()

    def _basic_transaction(self, row, obj_dict, child_dict):
        if len(row["description"]) > 255:
            obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
        return obj_dict, None, None

    def _template_transaction(self, row, obj_dict, child_dict):
        if self.buttons.get(row["description"]):
            child_dict["category_id"] = self.buttons[row["description"]][1]
            child_dict["template_id"] = self.buttons[row["description"]][0]
        # elif self.categories.get(row["categorie"]):
        #     child_dict["category_id"] = self.categories[row["categorie"]]
        elif "WEI" in row["description"]:
            return obj_dict, None, None
        else:
            return obj_dict, None, None
        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
        return obj_dict, child_dict, RecurrentTransaction

    def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
        obj_dict2 = obj_dict.copy()
        child_dict2 = child_dict.copy()
        child_dict2["membership_id"] = pk_membership

        return obj_dict2, child_dict2, MembershipTransaction

    def _special_transaction(self, row, obj_dict, child_dict):
        # Some transaction uses BDE (idbde=0) as source or destination,
        # lets fix that.
        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
        field_id = "source_id" if row["type"] == "crédit" else "destination_id"
        if "espèce" in row["description"]:
            obj_dict[field_id] = 1
        elif "carte" in row["description"]:
            obj_dict[field_id] = 2
        elif "cheques" in row["description"]:
            obj_dict[field_id] = 3
        elif "virement" in row["description"]:
            obj_dict[field_id] = 4
        # humans and clubs have always the biggest id
        actor_pk = max(row["destinataire"], row["emetteur"])
        actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
        # custom fields of SpecialTransaction
        if actor.__class__.__name__ == "NoteUser":
            child_dict["first_name"] = actor.user.first_name
            child_dict["last_name"] = actor.user.last_name
        else:
            child_dict["first_name"] = actor.club.name
            child_dict["last_name"] = actor.club.name
        return obj_dict, child_dict, SpecialTransaction

    def _guest_transaction(self, row, obj_dict, child_dict):
        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
        m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
        if m:
            activity_name = m.group(1)
            first_name, last_name = m.group(2), m.group(3)
            if first_name == "Marion" and last_name == "Bizu Pose":
                first_name, last_name = "Marion Bizu", "Pose"
            entry_id = Entry.objects.filter(
                    activity__name__iexact=activity_name,
                    guest__first_name__iexact=first_name,
                    guest__last_name__iexact=last_name,
            ).first().pk
            child_dict["entry_id"] = entry_id
        else:
            raise Exception(f"Guest not Found {row['id']} first_name, last_name")

        return obj_dict, child_dict, GuestTransaction

    @timed
    @transaction.atomic
    def import_transaction(self, cur, chunk_size, idmin, save=True):
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute(
            f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
              t.destinataire,t.quantite, t.montant, t.description,\
              t.valide, t.cantinvalidate, t.categorie, \
              a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
            FROM transactions AS t \
            LEFT JOIN adhesions  AS a ON t.id = a.idtransaction \
            WHERE t.id >= {idmin} \
            ORDER BY t.id;")
        n = cur.rowcount
        pk_membership = 1
        pk_transaction = 1
        kfet_balance = 0
        for idx, row in enumerate(cur):
            if save or idx % chunk_size == 0:
                self.update_line(idx, n, row["description"])

            MAP_TRANSACTION[row["id"]] = pk_transaction

            if not save:
                pk_transaction += 1
                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
                    if not isinstance(note, NoteClub):
                        pk_transaction += 1
                continue

            try:
                date = make_aware(row["transac_date"])
            except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
                date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))

            # standart transaction object
            obj_dict = {
                "pk": pk_transaction,
                "destination_id": self.MAP_IDBDE[row["destinataire"]],
                "polymorphic_ctype": None,
                "source_id": self.MAP_IDBDE[row["emetteur"]],
                "amount": row["montant"],
                "created_at": date,
                "destination_alias": "",
                "invalidity_reason": None,
                "quantity": row["quantite"],
                "reason": row["description"],
                "source_alias": "",
                "valid": row["valide"],
            }
            if len(obj_dict["reason"]) > 255:
                obj_dict["reason"] = obj_dict["reason"][:252] + "..."
            # for child transaction Models
            child_dict = {"pk": pk_transaction}
            ttype = row["type"]
            # Membership transaction detection and import
            if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                note = Note.objects.get(pk=obj_dict["source_id"])
                if isinstance(note, NoteClub):
                    child_transaction = None    # don't bother register clubs
                else:
                    user_id = note.user_id
                    montant = obj_dict["amount"]
                    (obj_dict0,
                     child_dict0,
                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                    obj_dict0["destination_id"] = 6  # Kfet note id
                    bde_dict = {
                        "pk": pk_membership,
                        "user_id": user_id,
                        "club_id": BDE_PK,
                        "date_start": date.date(),  # Only date, not time
                        "date_end": get_date_end(date.date()),
                        "fee": min(500, montant)
                    }
                    pk_membership += 1
                    pk_transaction += 1
                    obj_dict, child_dict, child_transaction =\
                        self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                    # Kfet membership
                    # BDE Membership
                    obj_dict["pk"] = pk_transaction
                    child_dict["pk"] = pk_transaction
                    kfet_dict = {
                        "pk": pk_membership,
                        "user_id": user_id,
                        "club_id": KFET_PK,
                        "date_start": date.date(),  # Only date, not time
                        "date_end": get_date_end(date.date()),
                        "fee": max(montant - 500, 0),
                    }
                    obj_dict0["amount"] = bde_dict["fee"]
                    obj_dict["amount"] = kfet_dict["fee"]
                    kfet_balance += kfet_dict["fee"]
                    # BDE membership Transaction is inserted before the Kfet membershipTransaction
                    pk_membership += 1
                    pk_transaction += 1
                    bulk_mgr.add(
                        Membership(**bde_dict),
                        Membership(**kfet_dict),
                        Transaction(**obj_dict0),
                        child_transaction(**child_dict0),
                        Transaction(**obj_dict),
                        child_transaction(**child_dict),
                    )
                    continue
            elif ttype == "bouton":
                obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
            elif ttype == "crédit" or ttype == "retrait":
                obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
            elif ttype == "invitation":
                obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
            elif ttype == "don" or ttype == "transfert":
                obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
            else:
                child_transaction = None
                # create base transaction object and typed one
            bulk_mgr.add(Transaction(**obj_dict))
            if child_transaction is not None:
                child_dict.update(obj_dict)
                bulk_mgr.add(child_transaction(**child_dict))
            pk_transaction += 1
        bulk_mgr.done()

        # Update the balance of the BDE and the Kfet club
        note_bde = NoteClub.objects.get(pk=5)
        note_kfet = NoteClub.objects.get(pk=6)
        note_bde.balance -= kfet_balance
        note_kfet.balance += kfet_balance
        note_bde.save()
        note_kfet.save()

    @timed
    def set_roles(self):
        bulk_mgr = BulkCreateManager(chunk_size=10000)
        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
        n = len(bde_membership_ids)
        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
            self.update_line(idx, n, str(idx))
            bulk_mgr.add(
                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
            )
            bulk_mgr.done()

    @timed
    @transaction.atomic
    def import_remittances(self, cur, chunk_size):
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
        n = cur.rowcount
        pk_remittance = 1

        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["commentaire"])

            MAP_REMITTANCE[row["id"]] = pk_remittance

            remittance_dict = {
                "pk": pk_remittance,
                "date": make_aware(row["date"]),
                "remittance_type_id": 1,    # Only Bank checks are supported in NK15
                "comment": row["commentaire"],
                "closed": row["close"],
            }
            bulk_mgr.add(Remittance(**remittance_dict))

            pk_remittance += 1

        bulk_mgr.done()

    @timed
    def import_checks(self, cur):
        cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
                    "FROM cheques ORDER BY id;")
        n = cur.rowcount

        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["nom"])

            if not row["idremise"]:
                continue

            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
            tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
            tr.save()

            tr = tr.transaction
            tr.last_name = row["nom"]
            tr.first_name = row["prenom"]
            tr.bank = row["banque"]
            try:
                tr.save()
            except:
                print("Failed to save row: " + str(row))

    @timed
    def import_soge_credits(self):
        users = User.objects.filter(profile__registration_valid=True).order_by('pk')
        n = users.count()
        for idx, user in enumerate(users.all()):
            self.update_line(idx, n, user.username)
            soge_credit_transaction = SpecialTransaction.objects.filter(
                reason__icontains="crédit sogé",
                destination_id=user.note.id,
            )
            if soge_credit_transaction.exists():
                soge_credit_transaction = soge_credit_transaction.get()
                soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
                memberships = Membership.objects.filter(
                    user=user,
                    club_id__in=[BDE_PK, KFET_PK],
                    date_start__lte=soge_credit_transaction.created_at,
                    date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
                ).all()
                for membership in memberships:
                    soge_credit.transactions.add(membership.transaction)
                soge_credit.save()


    @timed
    def handle(self, *args, **kwargs):
        # default args, provided by ImportCommand.
        nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
        # connecting to nk15 database
        conn = pg.connect(database=nk15db, user=nk15user)
        cur = conn.cursor(cursor_factory=pge.DictCursor)

        if kwargs["map"]:
            self.load_map(kwargs["map"])
        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
        if not kwargs["nosave"]:
            self.set_roles()
        self.import_remittances(cur, kwargs["chunk"])
        self.import_checks(cur)
        self.import_soge_credits()