# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay # SPDX-License-Identifier: GPL-3.0-or-later import copy import datetime import re import pytz import psycopg2 as pg import psycopg2.extras as pge from activity.models import Entry, GuestTransaction from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import transaction from django.utils.timezone import make_aware from member.models import Membership from note.models import (MembershipTransaction, Note, NoteClub, RecurrentTransaction, SpecialTransaction, TemplateCategory, Transaction, TransactionTemplate) from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy from ._import_utils import BulkCreateManager, ImportCommand, timed MAP_TRANSACTION = dict() MAP_REMITTANCE = dict() # from member/fixtures/initial BDE_PK = 1 KFET_PK = 2 # from note/fixtures/initial NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } # from permission/fixtures/initial BDE_ROLE_PK = 1 KFET_ROLE_PK = 2 CT = { "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"), "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"), "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"), "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"), } def get_date_end(date_start): date_end = copy.deepcopy(date_start) if date_start.month >= 8: date_end = date_start.replace(year=date_start.year + 1) date_end = date_end.replace(month=9, day=30) return date_end class Command(ImportCommand): """ Import command for People base data (Comptes, and Aliases) """ def add_arguments(self, parser): parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, " "don't save them") @timed def import_buttons(self, cur, chunk_size, import_buttons): self.categories = dict() self.buttons = dict() bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT * FROM boutons;") n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, row["label"]) if row["categorie"] not in self.categories: cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0] cat.save() self.categories[row["categorie"]] = cat.pk obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": self.MAP_IDBDE[row["destinataire"]], "category_id": self.categories[row["categorie"]], "display": row["affiche"], "description": row["description"], } if row["label"] in self.buttons: obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}" if import_buttons: bulk_mgr.add(TransactionTemplate(**obj_dict)) self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]]) bulk_mgr.done() def _basic_transaction(self, row, obj_dict, child_dict): if len(row["description"]) > 255: obj_dict["reason"] = obj_dict["reason"][:250] + "...)" return obj_dict, None, None def _template_transaction(self, row, obj_dict, child_dict): if self.buttons.get(row["description"]): child_dict["template_id"] = self.buttons[row["description"]][0] # elif self.categories.get(row["categorie"]): # child_dict["category_id"] = self.categories[row["categorie"]] elif "WEI" in row["description"]: return obj_dict, None, None else: return obj_dict, None, None obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"] return obj_dict, child_dict, RecurrentTransaction def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"] obj_dict2 = obj_dict.copy() child_dict2 = child_dict.copy() child_dict2["membership_id"] = pk_membership return obj_dict2, child_dict2, MembershipTransaction def _special_transaction(self, row, obj_dict, child_dict): # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"] field_id = "source_id" if row["type"] == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 elif "carte" in row["description"]: obj_dict[field_id] = 2 elif "cheques" in row["description"]: obj_dict[field_id] = 3 elif "virement" in row["description"]: obj_dict[field_id] = 4 # humans and clubs have always the biggest id actor_pk = max(row["destinataire"], row["emetteur"]) actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) # custom fields of SpecialTransaction if actor.__class__.__name__ == "NoteUser": child_dict["first_name"] = actor.user.first_name child_dict["last_name"] = actor.user.last_name else: child_dict["first_name"] = actor.club.name child_dict["last_name"] = actor.club.name return obj_dict, child_dict, SpecialTransaction def _guest_transaction(self, row, obj_dict, child_dict): obj_dict["polymorphic_ctype"] = CT["GuestTransaction"] m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: activity_name = m.group(1) first_name, last_name = m.group(2), m.group(3) if first_name == "Marion" and last_name == "Bizu Pose": first_name, last_name = "Marion Bizu", "Pose" entry_id = Entry.objects.filter( activity__name__iexact=activity_name, guest__first_name__iexact=first_name, guest__last_name__iexact=last_name, ).first().pk child_dict["entry_id"] = entry_id else: raise Exception(f"Guest not Found {row['id']} first_name, last_name") return obj_dict, child_dict, GuestTransaction @timed @transaction.atomic def import_transaction(self, cur, chunk_size, idmin, save=True): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\ t.destinataire,t.quantite, t.montant, t.description,\ t.valide, t.cantinvalidate, t.categorie, \ a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ FROM transactions AS t \ LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ WHERE t.id >= {idmin} \ ORDER BY t.id;") n = cur.rowcount pk_membership = 1 pk_transaction = 1 kfet_balance = 0 for idx, row in enumerate(cur): if save or idx % chunk_size == 0: self.update_line(idx, n, row["description"]) MAP_TRANSACTION[row["id"]] = pk_transaction if not save: pk_transaction += 1 if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]]) if not isinstance(note, NoteClub): pk_transaction += 1 continue try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) # standart transaction object obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], "polymorphic_ctype": None, "source_id": self.MAP_IDBDE[row["emetteur"]], "amount": row["montant"], "created_at": date, "destination_alias": "", "invalidity_reason": "", "quantity": row["quantite"], "reason": row["description"], "source_alias": "", "valid": row["valide"], } if len(obj_dict["reason"]) > 255: obj_dict["reason"] = obj_dict["reason"][:252] + "..." # for child transaction Models child_dict = {"pk": pk_transaction} ttype = row["type"] # Membership transaction detection and import if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): child_transaction = None # don't bother register clubs else: user_id = note.user_id montant = obj_dict["amount"] (obj_dict0, child_dict0, child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership) obj_dict0["destination_id"] = 6 # Kfet note id bde_dict = { "pk": pk_membership, "user_id": user_id, "club_id": BDE_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 obj_dict, child_dict, child_transaction =\ self._membership_transaction(row, obj_dict, child_dict, pk_membership) # Kfet membership # BDE Membership obj_dict["pk"] = pk_transaction child_dict["pk"] = pk_transaction kfet_dict = { "pk": pk_membership, "user_id": user_id, "club_id": KFET_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": max(montant - 500, 0), } obj_dict0["amount"] = bde_dict["fee"] obj_dict["amount"] = kfet_dict["fee"] kfet_balance += kfet_dict["fee"] # BDE membership Transaction is inserted before the Kfet membershipTransaction pk_membership += 1 pk_transaction += 1 bulk_mgr.add( Membership(**bde_dict), Membership(**kfet_dict), Transaction(**obj_dict0), child_transaction(**child_dict0), Transaction(**obj_dict), child_transaction(**child_dict), ) continue elif ttype == "bouton": obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict) elif ttype == "crédit" or ttype == "retrait": obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) elif ttype == "don" or ttype == "transfert": obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) else: child_transaction = None # create base transaction object and typed one bulk_mgr.add(Transaction(**obj_dict)) if child_transaction is not None: child_dict.update(obj_dict) bulk_mgr.add(child_transaction(**child_dict)) pk_transaction += 1 bulk_mgr.done() # Update the balance of the BDE and the Kfet club note_bde = NoteClub.objects.get(pk=5) note_kfet = NoteClub.objects.get(pk=6) note_bde.balance -= kfet_balance note_kfet.balance += kfet_balance note_bde.save() note_kfet.save() @timed def set_roles(self): bulk_mgr = BulkCreateManager(chunk_size=10000) bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True) kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True) n = len(bde_membership_ids) for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)): self.update_line(idx, n, str(idx)) bulk_mgr.add( Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK), Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK), ) bulk_mgr.done() # Note account has a different treatment for m in Membership.objects.filter(user_username="note").all(): m.date_end = "3142-12-12" m.roles.set([20]) # PC Kfet role m.save() @timed @transaction.atomic def import_remittances(self, cur, chunk_size): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;") n = cur.rowcount pk_remittance = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["commentaire"]) MAP_REMITTANCE[row["id"]] = pk_remittance remittance_dict = { "pk": pk_remittance, "date": make_aware(row["date"]), "remittance_type_id": 1, # Only Bank checks are supported in NK15 "comment": row["commentaire"], "closed": row["close"], } bulk_mgr.add(Remittance(**remittance_dict)) pk_remittance += 1 bulk_mgr.done() @timed def import_checks(self, cur): cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise " "FROM cheques ORDER BY id;") n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, row["nom"]) if not row["idremise"]: continue tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0] tr.remittance_id = MAP_REMITTANCE[row["idremise"]] tr.save() tr = tr.transaction tr.last_name = row["nom"] tr.first_name = row["prenom"] tr.bank = row["banque"] try: tr.save() except: print("Failed to save row: " + str(row)) @timed def import_soge_credits(self): users = User.objects.filter(profile__registration_valid=True).order_by('pk') n = users.count() for idx, user in enumerate(users.all()): self.update_line(idx, n, user.username) soge_credit_transaction = SpecialTransaction.objects.filter( reason__icontains="crédit sogé", destination_id=user.note.id, ) if soge_credit_transaction.exists(): soge_credit_transaction = soge_credit_transaction.get() soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction) memberships = Membership.objects.filter( user=user, club_id__in=[BDE_PK, KFET_PK], date_start__lte=soge_credit_transaction.created_at, date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61), ).all() for membership in memberships: soge_credit.transactions.add(membership.transaction) soge_credit.save() @timed def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["map"]: self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"]) if not kwargs["nosave"]: self.set_roles() self.import_remittances(cur, kwargs["chunk"]) self.import_checks(cur) self.import_soge_credits()