nk20-scripts/management/commands/import_transaction.py

# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
# SPDX-License-Identifier: GPL-3.0-or-later

import copy
import datetime
import re

import pytz
import psycopg2 as pg
import psycopg2.extras as pge
from activity.models import Entry, GuestTransaction
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import transaction
from django.utils.timezone import make_aware
from member.models import Membership
from note.models import (MembershipTransaction, Note, NoteClub,
                         RecurrentTransaction, SpecialTransaction,
                         TemplateCategory, Transaction, TransactionTemplate)
from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy

from ._import_utils import BulkCreateManager, ImportCommand, timed

MAP_TRANSACTION = dict()
MAP_REMITTANCE = dict()

# from member/fixtures/initial
BDE_PK = 1
KFET_PK = 2

# from note/fixtures/initial
NOTE_SPECIAL_CODE = {
    "espèce": 1,
    "carte": 2,
    "chèque": 3,
    "virement": 4,
}
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2

CT = {
    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}

def get_date_end(date_start):
    date_end = copy.deepcopy(date_start)
    if date_start.month >= 8:
        date_end = date_start.replace(year=date_start.year + 1)
    date_end = date_end.replace(month=9, day=30)
    return date_end


class Command(ImportCommand):
    """
    Import command for People base data (Comptes, and Aliases)
    """

    def add_arguments(self, parser):
        parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
        parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
        parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
                                                                                       "don't save them")

    @timed
    def import_buttons(self, cur, chunk_size, import_buttons):
        self.categories = dict()
        self.buttons = dict()
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute("SELECT * FROM boutons;")
        n = cur.rowcount
        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["label"])
            if row["categorie"] not in self.categories:
                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
                cat.save()
                self.categories[row["categorie"]] = cat.pk
            obj_dict = {
                "pk": row["id"],
                "name": row["label"],
                "amount": row["montant"],
                "destination_id": self.MAP_IDBDE[row["destinataire"]],
                "category_id": self.categories[row["categorie"]],
                "display": row["affiche"],
                "description": row["description"],
            }
            if row["label"] in self.buttons:
                obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
            if import_buttons:
                bulk_mgr.add(TransactionTemplate(**obj_dict))
            self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
        bulk_mgr.done()

    def _basic_transaction(self, row, obj_dict, child_dict):
        if len(row["description"]) > 255:
            obj_dict["reason"] = obj_dict["reason"][:252] + "…)"
        return obj_dict, None, None

    def _template_transaction(self, row, obj_dict, child_dict):
        if self.buttons.get(row["description"]):
            child_dict["template_id"] = self.buttons[row["description"]][0]
        # elif self.categories.get(row["categorie"]):
        #     child_dict["category_id"] = self.categories[row["categorie"]]
        elif "WEI" in row["description"]:
            return obj_dict, None, None
        else:
            return obj_dict, None, None
        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
        return obj_dict, child_dict, RecurrentTransaction

    def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
        obj_dict2 = obj_dict.copy()
        child_dict2 = child_dict.copy()
        child_dict2["membership_id"] = pk_membership

        return obj_dict2, child_dict2, MembershipTransaction

    def _special_transaction(self, row, obj_dict, child_dict):
        # Some transaction uses BDE (idbde=0) as source or destination,
        # lets fix that.
        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
        field_id = "source_id" if row["type"] == "crédit" else "destination_id"
        if "espèce" in row["description"]:
            obj_dict[field_id] = 1
        elif "carte" in row["description"]:
            obj_dict[field_id] = 2
        elif "cheques" in row["description"]:
            obj_dict[field_id] = 3
        elif "virement" in row["description"]:
            obj_dict[field_id] = 4
        # humans and clubs have always the biggest id
        actor_pk = max(row["destinataire"], row["emetteur"])
        actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
        # custom fields of SpecialTransaction
        if actor.__class__.__name__ == "NoteUser":
            child_dict["first_name"] = actor.user.first_name
            child_dict["last_name"] = actor.user.last_name
        else:
            child_dict["first_name"] = actor.club.name
            child_dict["last_name"] = actor.club.name
        return obj_dict, child_dict, SpecialTransaction

    def _guest_transaction(self, row, obj_dict, child_dict):
        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
        m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
        if m:
            activity_name = m.group(1)
            first_name, last_name = m.group(2), m.group(3)
            if first_name == "Marion" and last_name == "Bizu Pose":
                first_name, last_name = "Marion Bizu", "Pose"
            entry_id = Entry.objects.filter(
                    activity__name__iexact=activity_name,
                    guest__first_name__iexact=first_name,
                    guest__last_name__iexact=last_name,
            ).first().pk
            child_dict["entry_id"] = entry_id
        else:
            raise Exception(f"Guest not Found {row['id']} first_name, last_name")

        return obj_dict, child_dict, GuestTransaction

    @timed
    @transaction.atomic
    def import_transaction(self, cur, chunk_size, idmin, save=True):
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute(
            f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
              t.destinataire,t.quantite, t.montant, t.description,\
              t.valide, t.cantinvalidate, t.categorie, \
              a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
            FROM transactions AS t \
            LEFT JOIN adhesions  AS a ON t.id = a.idtransaction \
            WHERE t.id >= {idmin} \
            ORDER BY t.id;")
        n = cur.rowcount
        pk_membership = 1
        pk_transaction = 1
        kfet_balance = 0
        for idx, row in enumerate(cur):
            if save or idx % chunk_size == 0:
                self.update_line(idx, n, row["description"])

            MAP_TRANSACTION[row["id"]] = pk_transaction

            if not save:
                pk_transaction += 1
                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
                    if not isinstance(note, NoteClub):
                        pk_transaction += 1
                continue

            try:
                date = make_aware(row["transac_date"])
            except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
                date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))

            # standart transaction object
            obj_dict = {
                "pk": pk_transaction,
                "destination_id": self.MAP_IDBDE[row["destinataire"]],
                "polymorphic_ctype": None,
                "source_id": self.MAP_IDBDE[row["emetteur"]],
                "amount": row["montant"],
                "created_at": date,
                "destination_alias": "",
                "invalidity_reason": "",
                "quantity": row["quantite"],
                "reason": row["description"],
                "source_alias": "",
                "valid": row["valide"],
            }
            if len(obj_dict["reason"]) > 255:
                obj_dict["reason"] = obj_dict["reason"][:254] + "…"
            # for child transaction Models
            child_dict = {"pk": pk_transaction}
            ttype = row["type"]
            # Membership transaction detection and import
            if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                note = Note.objects.get(pk=obj_dict["source_id"])
                if isinstance(note, NoteClub):
                    child_transaction = None    # don't bother register clubs
                else:
                    user_id = note.user_id
                    montant = obj_dict["amount"]
                    (obj_dict0,
                     child_dict0,
                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                    obj_dict0["destination_id"] = 6  # Kfet note id
                    bde_dict = {
                        "pk": pk_membership,
                        "user_id": user_id,
                        "club_id": BDE_PK,
                        "date_start": date.date(),  # Only date, not time
                        "date_end": get_date_end(date.date()),
                        "fee": min(500, montant)
                    }
                    pk_membership += 1
                    pk_transaction += 1
                    obj_dict, child_dict, child_transaction =\
                        self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                    # Kfet membership
                    # BDE Membership
                    obj_dict["pk"] = pk_transaction
                    child_dict["pk"] = pk_transaction
                    kfet_dict = {
                        "pk": pk_membership,
                        "user_id": user_id,
                        "club_id": KFET_PK,
                        "date_start": date.date(),  # Only date, not time
                        "date_end": get_date_end(date.date()),
                        "fee": max(montant - 500, 0),
                    }
                    obj_dict0["amount"] = bde_dict["fee"]
                    obj_dict["amount"] = kfet_dict["fee"]
                    kfet_balance += kfet_dict["fee"]
                    # BDE membership Transaction is inserted before the Kfet membershipTransaction
                    pk_membership += 1
                    pk_transaction += 1
                    bulk_mgr.add(
                        Membership(**bde_dict),
                        Membership(**kfet_dict),
                        Transaction(**obj_dict0),
                        child_transaction(**child_dict0),
                        Transaction(**obj_dict),
                        child_transaction(**child_dict),
                    )
                    continue
            elif ttype == "bouton":
                obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
            elif ttype == "crédit" or ttype == "retrait":
                obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
            elif ttype == "invitation":
                obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
            elif ttype == "don" or ttype == "transfert":
                obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
            else:
                child_transaction = None
                # create base transaction object and typed one
            bulk_mgr.add(Transaction(**obj_dict))
            if child_transaction is not None:
                child_dict.update(obj_dict)
                bulk_mgr.add(child_transaction(**child_dict))
            pk_transaction += 1
        bulk_mgr.done()

        # Update the balance of the BDE and the Kfet club
        note_bde = NoteClub.objects.get(pk=5)
        note_kfet = NoteClub.objects.get(pk=6)
        note_bde.balance -= kfet_balance
        note_kfet.balance += kfet_balance
        note_bde.save()
        note_kfet.save()

    @timed
    def set_roles(self):
        bulk_mgr = BulkCreateManager(chunk_size=10000)
        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
        n = len(bde_membership_ids)
        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
            self.update_line(idx, n, str(idx))
            bulk_mgr.add(
                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
            )
            bulk_mgr.done()

        # Note account has a different treatment
        for m in Membership.objects.filter(user_username="note").all():
            m.date_end = "3142-12-12"
            m.roles.set([20])   # PC Kfet role
            m.save()

    @timed
    @transaction.atomic
    def import_remittances(self, cur, chunk_size):
        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
        n = cur.rowcount
        pk_remittance = 1

        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["commentaire"])

            MAP_REMITTANCE[row["id"]] = pk_remittance

            remittance_dict = {
                "pk": pk_remittance,
                "date": make_aware(row["date"]),
                "remittance_type_id": 1,    # Only Bank checks are supported in NK15
                "comment": row["commentaire"],
                "closed": row["close"],
            }
            bulk_mgr.add(Remittance(**remittance_dict))

            pk_remittance += 1

        bulk_mgr.done()

    @timed
    def import_checks(self, cur):
        cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
                    "FROM cheques ORDER BY id;")
        n = cur.rowcount

        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["nom"])

            if not row["idremise"]:
                continue

            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
            tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
            tr.save()

            tr = tr.transaction
            tr.last_name = row["nom"]
            tr.first_name = row["prenom"]
            tr.bank = row["banque"]
            try:
                tr.save()
            except:
                print("Failed to save row: " + str(row))

    @timed
    def import_soge_credits(self):
        users = User.objects.filter(profile__registration_valid=True).order_by('pk')
        n = users.count()
        for idx, user in enumerate(users.all()):
            self.update_line(idx, n, user.username)
            soge_credit_transaction = SpecialTransaction.objects.filter(
                reason__icontains="crédit sogé",
                destination_id=user.note.id,
            )
            if soge_credit_transaction.exists():
                soge_credit_transaction = soge_credit_transaction.get()
                soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
                memberships = Membership.objects.filter(
                    user=user,
                    club_id__in=[BDE_PK, KFET_PK],
                    date_start__lte=soge_credit_transaction.created_at,
                    date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
                ).all()
                for membership in memberships:
                    soge_credit.transactions.add(membership.transaction)
                soge_credit.save()


    @timed
    def handle(self, *args, **kwargs):
        # default args, provided by ImportCommand.
        nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
        # connecting to nk15 database
        conn = pg.connect(database=nk15db, user=nk15user)
        cur = conn.cursor(cursor_factory=pge.DictCursor)

        if kwargs["map"]:
            self.load_map(kwargs["map"])
        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
        if not kwargs["nosave"]:
            self.set_roles()
        self.import_remittances(cur, kwargs["chunk"])
        self.import_checks(cur)
        self.import_soge_credits()