#!/usr/bin/env python3 import re import psycopg2 as pg import psycopg2.extras as pge import pytz import datetime import copy from django.utils.timezone import make_aware from django.db import transaction from note.models import (TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, SpecialTransaction ) from note.models import Note, NoteClub from activity.models import Guest, GuestTransaction from member.models import Membership, MembershipTransaction, Role from ._import_utils import ImportCommand, BulkCreateManager, timed BDE_PK = 1 KFET_PK = 2 NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } def get_date_end(date_start): date_end = copy.deepcopy(date_start) if date_start.month > 8: date_end = date_start.replace(year=date_start.year+1) date_end = date_end.replace(month=9, day=30) return date_end class Command(ImportCommand): """ Import command for People base data (Comptes, and Aliases) """ def add_arguments(self, parser): parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") @timed def import_buttons(self, cur, chunk_size, import_buttons): self.categories = dict() self.buttons = dict() bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT * FROM boutons;") n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, row["label"]) if row["categorie"] not in self.categories: cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0] cat.save() self.categories[row["categorie"]] = cat.pk obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": self.MAP_IDBDE[row["destinataire"]], "category_id": self.categories[row["categorie"]], "display": row["affiche"], "description": row["description"], } if row["label"] in self.buttons: obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}" if import_buttons: bulk_mgr.add(TransactionTemplate(**obj_dict)) self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]]) bulk_mgr.done() def _basic_transaction(self, row, obj_dict, child_dict): if len(row["description"]) > 255: obj_dict["reason"] = obj_dict["reason"][:250]+"...)" return obj_dict, None, None def _template_transaction(self, row, obj_dict, child_dict): if self.categories.get(row["categorie"]): child_dict["category_id"] = self.categories[row["categorie"]] elif "WEI" in row["description"]: return obj_dict, None, None elif self.buttons.get(row["description"]): child_dict["category_id"] = self.buttons[row["description"]][1] child_dict["template_id"] = self.buttons[row["description"]][0] else: return obj_dict, None, None return obj_dict, child_dict, RecurrentTransaction def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): obj_dict2 = obj_dict.copy() child_dict2 = child_dict.copy() child_dict2["membership_id"] = pk_membership return obj_dict2, child_dict2, MembershipTransaction def _special_transaction(self, row, obj_dict, child_dict): # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. field_id = "source_id" if row["type"] == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 elif "carte" in row["description"]: obj_dict[field_id] = 2 elif "cheques" in row["description"]: obj_dict[field_id] = 3 elif "virement" in row["description"]: obj_dict[field_id] = 4 # humans and clubs have always the biggest id actor_pk = max(row["destinataire"], row["emetteur"]) actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) # custom fields of SpecialTransaction if actor.__class__.__name__ == "NoteUser": child_dict["first_name"] = actor.user.first_name child_dict["last_name"] = actor.user.last_name else: child_dict["first_name"] = actor.club.name child_dict["last_name"] = actor.club.name return obj_dict, child_dict, SpecialTransaction def _guest_transaction(self, row, obj_dict, child_dict): # Currently GuestTransaction is related to a Guest. # This is not ideal and should be change to the Entry of this Guest. m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: first_name, last_name = m.group(2), m.group(3) if first_name == "Marion" and last_name == "Bizu Pose": first_name, last_name = "Marion Bizu", "Pose" guest_id = Guest.objects.filter(first_name__iexact=first_name, last_name__iexact=last_name).first().pk child_dict["guest_id"] = guest_id else: raise(f"Guest not Found {row['id']} {first_name}, last_name") return obj_dict, child_dict, GuestTransaction @timed @transaction.atomic def import_transaction(self, cur, chunk_size, idmin): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.date AS transac_date, t.type, t.emetteur,\ t.destinataire,t.quantite, t.montant, t.description,\ t.valide, t.cantinvalidate, t.categorie, \ a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ FROM transactions AS t \ LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ WHERE t.id >= {idmin} \ ORDER BY t.id;") n = cur.rowcount pk_membership = 1 pk_transaction = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["description"]) try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) if len(row["description"]) > 255: row["description"] = row["description"][:252] + "..." # standart transaction object obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], "source_id": self.MAP_IDBDE[row["emetteur"]], "created_at": date, "amount": row["montant"], "quantity": row["quantite"], "reason": row["description"], "valid": row["valide"], } # for child transaction Models child_dict = {"pk": obj_dict["pk"]} ttype = row["type"] if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): child_transaction = None else: user_id = note.user_id montant = obj_dict["amount"] obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) bde_dict = { "pk": pk_membership, "user_id": user_id, "club_id": KFET_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) # Kfet membership # BDE Membership obj_dict["pk"] = pk_transaction child_dict["pk"] = pk_transaction kfet_dict = { "pk": pk_membership, "user_id": user_id, "club_id": BDE_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": max(montant - 500, 0), } obj_dict0["amount"] = bde_dict["fee"] obj_dict["amount"] = kfet_dict["fee"] # BDE membership Transaction is inserted before the Kfet membershipTransaction pk_membership += 1 pk_transaction += 1 bulk_mgr.add( Transaction(**obj_dict0), child_transaction(**child_dict0), Transaction(**obj_dict), child_transaction(**child_dict), Membership(**bde_dict), Membership(**kfet_dict), ) continue elif ttype == "bouton": obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict) elif ttype == "crédit" or ttype == "retrait": obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) if ttype == "don" or ttype == "transfert": obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) else: child_transaction = None # create base transaction object and typed one bulk_mgr.add(Transaction(**obj_dict)) if child_transaction is not None: bulk_mgr.add(child_transaction(**child_dict)) pk_transaction += 1 bulk_mgr.done() @timed def adjust_roles(self): bdeRole = Role.objects.get(name="Adhérent BDE") kfetRole = Role.objects.get(name="Adhérent Kfet") memberships = Membership.objects.all() n = len(memberships) for idx, membership in enumerate(memberships): self.update_line(idx, n, membership.user.username) if membership.club.name == "BDE": membership.roles.set([bdeRole]) elif membership.club.name == "Kfet": membership.roles.set([kfetRole]) @timed def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["map"]: self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"]) self.adjust_roles()