# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay # SPDX-License-Identifier: GPL-3.0-or-later import re import psycopg2 as pg import psycopg2.extras as pge import pytz import datetime import copy from django.contrib.auth.models import User from django.utils.timezone import make_aware from django.db import transaction from django.contrib.contenttypes.models import ContentType from note.models import (TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, SpecialTransaction, MembershipTransaction, ) from note.models import Note, NoteClub from activity.models import Guest, GuestTransaction, Entry from member.models import Membership from treasury.models import Remittance, SpecialTransactionProxy, SogeCredit from ._import_utils import ImportCommand, BulkCreateManager, timed MAP_TRANSACTION = dict() MAP_REMITTANCE = dict() # from member/fixtures/initial BDE_PK = 1 KFET_PK = 2 # from note/fixtures/initial NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } # from permission/fixtures/initial BDE_ROLE_PK = 1 KFET_ROLE_PK = 2 CT = { "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"), "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"), "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"), "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"), } def get_date_end(date_start): date_end = copy.deepcopy(date_start) if date_start.month >= 8: date_end = date_start.replace(year=date_start.year + 1) date_end = date_end.replace(month=9, day=30) return date_end class Command(ImportCommand): """ Import command for People base data (Comptes, and Aliases) """ def add_arguments(self, parser): parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, " "don't save them") @timed def import_buttons(self, cur, chunk_size, import_buttons): self.categories = dict() self.buttons = dict() bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT * FROM boutons;") n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, row["label"]) if row["categorie"] not in self.categories: cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0] cat.save() self.categories[row["categorie"]] = cat.pk obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": self.MAP_IDBDE[row["destinataire"]], "category_id": self.categories[row["categorie"]], "display": row["affiche"], "description": row["description"], } if row["label"] in self.buttons: obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}" if import_buttons: bulk_mgr.add(TransactionTemplate(**obj_dict)) self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]]) bulk_mgr.done() def _basic_transaction(self, row, obj_dict, child_dict): if len(row["description"]) > 255: obj_dict["reason"] = obj_dict["reason"][:250] + "...)" return obj_dict, None, None def _template_transaction(self, row, obj_dict, child_dict): if self.buttons.get(row["description"]): child_dict["template_id"] = self.buttons[row["description"]][0] # elif self.categories.get(row["categorie"]): # child_dict["category_id"] = self.categories[row["categorie"]] elif "WEI" in row["description"]: return obj_dict, None, None else: return obj_dict, None, None obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"] return obj_dict, child_dict, RecurrentTransaction def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"] obj_dict2 = obj_dict.copy() child_dict2 = child_dict.copy() child_dict2["membership_id"] = pk_membership return obj_dict2, child_dict2, MembershipTransaction def _special_transaction(self, row, obj_dict, child_dict): # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"] field_id = "source_id" if row["type"] == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 elif "carte" in row["description"]: obj_dict[field_id] = 2 elif "cheques" in row["description"]: obj_dict[field_id] = 3 elif "virement" in row["description"]: obj_dict[field_id] = 4 # humans and clubs have always the biggest id actor_pk = max(row["destinataire"], row["emetteur"]) actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) # custom fields of SpecialTransaction if actor.__class__.__name__ == "NoteUser": child_dict["first_name"] = actor.user.first_name child_dict["last_name"] = actor.user.last_name else: child_dict["first_name"] = actor.club.name child_dict["last_name"] = actor.club.name return obj_dict, child_dict, SpecialTransaction def _guest_transaction(self, row, obj_dict, child_dict): obj_dict["polymorphic_ctype"] = CT["GuestTransaction"] m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: activity_name = m.group(1) first_name, last_name = m.group(2), m.group(3) if first_name == "Marion" and last_name == "Bizu Pose": first_name, last_name = "Marion Bizu", "Pose" entry_id = Entry.objects.filter( activity__name__iexact=activity_name, guest__first_name__iexact=first_name, guest__last_name__iexact=last_name, ).first().pk child_dict["entry_id"] = entry_id else: raise Exception(f"Guest not Found {row['id']} first_name, last_name") return obj_dict, child_dict, GuestTransaction @timed @transaction.atomic def import_transaction(self, cur, chunk_size, idmin, save=True): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\ t.destinataire,t.quantite, t.montant, t.description,\ t.valide, t.cantinvalidate, t.categorie, \ a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ FROM transactions AS t \ LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ WHERE t.id >= {idmin} \ ORDER BY t.id;") n = cur.rowcount pk_membership = 1 pk_transaction = 1 kfet_balance = 0 for idx, row in enumerate(cur): if save or idx % chunk_size == 0: self.update_line(idx, n, row["description"]) MAP_TRANSACTION[row["id"]] = pk_transaction if not save: pk_transaction += 1 if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]]) if not isinstance(note, NoteClub): pk_transaction += 1 continue try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) # standart transaction object obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], "polymorphic_ctype": None, "source_id": self.MAP_IDBDE[row["emetteur"]], "amount": row["montant"], "created_at": date, "destination_alias": "", "invalidity_reason": "", "quantity": row["quantite"], "reason": row["description"], "source_alias": "", "valid": row["valide"], } if len(obj_dict["reason"]) > 255: obj_dict["reason"] = obj_dict["reason"][:252] + "..." # for child transaction Models child_dict = {"pk": pk_transaction} ttype = row["type"] # Membership transaction detection and import if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): child_transaction = None # don't bother register clubs else: user_id = note.user_id montant = obj_dict["amount"] (obj_dict0, child_dict0, child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership) obj_dict0["destination_id"] = 6 # Kfet note id bde_dict = { "pk": pk_membership, "user_id": user_id, "club_id": BDE_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 obj_dict, child_dict, child_transaction =\ self._membership_transaction(row, obj_dict, child_dict, pk_membership) # Kfet membership # BDE Membership obj_dict["pk"] = pk_transaction child_dict["pk"] = pk_transaction kfet_dict = { "pk": pk_membership, "user_id": user_id, "club_id": KFET_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": max(montant - 500, 0), } obj_dict0["amount"] = bde_dict["fee"] obj_dict["amount"] = kfet_dict["fee"] kfet_balance += kfet_dict["fee"] # BDE membership Transaction is inserted before the Kfet membershipTransaction pk_membership += 1 pk_transaction += 1 bulk_mgr.add( Membership(**bde_dict), Membership(**kfet_dict), Transaction(**obj_dict0), child_transaction(**child_dict0), Transaction(**obj_dict), child_transaction(**child_dict), ) continue elif ttype == "bouton": obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict) elif ttype == "crédit" or ttype == "retrait": obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) elif ttype == "don" or ttype == "transfert": obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) else: child_transaction = None # create base transaction object and typed one bulk_mgr.add(Transaction(**obj_dict)) if child_transaction is not None: child_dict.update(obj_dict) bulk_mgr.add(child_transaction(**child_dict)) pk_transaction += 1 bulk_mgr.done() # Update the balance of the BDE and the Kfet club note_bde = NoteClub.objects.get(pk=5) note_kfet = NoteClub.objects.get(pk=6) note_bde.balance -= kfet_balance note_kfet.balance += kfet_balance note_bde.save() note_kfet.save() @timed def set_roles(self): bulk_mgr = BulkCreateManager(chunk_size=10000) bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True) kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True) n = len(bde_membership_ids) for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)): self.update_line(idx, n, str(idx)) bulk_mgr.add( Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK), Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK), ) bulk_mgr.done() # Note account has a different treatment for m in Membership.objects.filter(user_username="note").all(): m.date_end = "3142-12-12" m.roles.set([20]) # PC Kfet role m.save() @timed @transaction.atomic def import_remittances(self, cur, chunk_size): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;") n = cur.rowcount pk_remittance = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["commentaire"]) MAP_REMITTANCE[row["id"]] = pk_remittance remittance_dict = { "pk": pk_remittance, "date": make_aware(row["date"]), "remittance_type_id": 1, # Only Bank checks are supported in NK15 "comment": row["commentaire"], "closed": row["close"], } bulk_mgr.add(Remittance(**remittance_dict)) pk_remittance += 1 bulk_mgr.done() @timed def import_checks(self, cur): cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise " "FROM cheques ORDER BY id;") n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, row["nom"]) if not row["idremise"]: continue tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0] tr.remittance_id = MAP_REMITTANCE[row["idremise"]] tr.save() tr = tr.transaction tr.last_name = row["nom"] tr.first_name = row["prenom"] tr.bank = row["banque"] try: tr.save() except: print("Failed to save row: " + str(row)) @timed def import_soge_credits(self): users = User.objects.filter(profile__registration_valid=True).order_by('pk') n = users.count() for idx, user in enumerate(users.all()): self.update_line(idx, n, user.username) soge_credit_transaction = SpecialTransaction.objects.filter( reason__icontains="crédit sogé", destination_id=user.note.id, ) if soge_credit_transaction.exists(): soge_credit_transaction = soge_credit_transaction.get() soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction) memberships = Membership.objects.filter( user=user, club_id__in=[BDE_PK, KFET_PK], date_start__lte=soge_credit_transaction.created_at, date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61), ).all() for membership in memberships: soge_credit.transactions.add(membership.transaction) soge_credit.save() @timed def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["map"]: self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"]) if not kwargs["nosave"]: self.set_roles() self.import_remittances(cur, kwargs["chunk"]) self.import_checks(cur) self.import_soge_credits()