#!/usr/bin/env python3 import re import psycopg2 as pg import psycopg2.extras as pge import pytz import datetime import copy from django.utils.timezone import make_aware from django.db import transaction from note.models import (TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, SpecialTransaction ) from note.models import Note from activity.models import Guest, GuestTransaction from member.models import Membership, MembershipTransaction from ._import_utils import ImportCommand, BulkCreateManager, timed BDE_PK = 1 KFET_PK = 2 NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } def get_date_end(date_start): date_end = copy.deepcopy(date_start) if date_start > 8: date_end.year = date_start + 1 date_end.month = 9 date_end.day = 30 return date_end class Command(ImportCommand): """ Import command for People base data (Comptes, and Aliases) """ def add_arguments(self, parser): parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") @timed @transaction.atomic def import_buttons(self, cur, chunk_size): categories = dict() buttons = dict() bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT * FROM boutons;") n = cur.rowcount pk_category = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["label"]) if row["categorie"] not in categories: bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"])) pk_category += 1 categories[row["categorie"]] = pk_category obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": self.MAP_IDBDE[row["destinataire"]], "category_id": categories[row["categorie"]], "display": row["affiche"], "description": row["description"], } if row["label"] in buttons: obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}" bulk_mgr.add(TransactionTemplate(**obj_dict)) buttons[obj_dict["label"]] = row["id"] bulk_mgr.done() return buttons, categories @timed @transaction.atomic def import_transaction(self, cur, chunk_size, idmin, buttons, categories): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.date AS transac_date, t.type, t.emetteur,\ t.destinataire,t.quantite, t.montant, t.description,\ t.valide, t.cantinvalidate, t.categorie, \ a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ FROM transactions AS t \ LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ WHERE t.id >= {idmin} \ ORDER BY t.id;") n = cur.rowcount pk_membership = 1 pk_transaction = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["description"]) try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) # standart transaction object obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], "source_id": self.MAP_IDBDE[row["emetteur"]], "created_at": date, "amount": row["montant"], "quantity": row["quantite"], "reason": row["description"], "valid": row["valide"], } # for child transaction Models child_dict = {"pk": obj_dict["pk"]} ttype = row["type"] if ttype == "don" or ttype == "transfert": child_transaction = None elif ttype == "bouton": child_transaction = RecurrentTransaction child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"]) child_dict["template_id"] = buttons[row["description"]] elif ttype == "crédit" or ttype == "retrait": child_transaction = SpecialTransaction # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. field_id = "source_id" if ttype == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 elif "carte" in row["description"]: obj_dict[field_id] = 2 elif "cheques" in row["description"]: obj_dict[field_id] = 3 elif "virement" in row["description"]: obj_dict[field_id] = 4 # humans and clubs have always the biggest id actor_pk = max(row["destinataire"], row["emetteur"]) actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) # custom fields of SpecialTransaction if actor.__class__.__name__ == "NoteUser": child_dict["first_name"] = actor.user.first_name child_dict["last_name"] = actor.user.last_name else: child_dict["first_name"] = actor.club.name child_dict["last_name"] = actor.club.name elif ttype == "adhésion" and row["valide"]: child_transaction = MembershipTransaction # Kfet membership montant = row["montant"] obj_dict["amount"] = min(500, montant) child_dict["membership_id"] = pk_membership kfet_dict = { "pk": pk_membership, "user": self.MAP_IDBDE[row["idbde"]], "club": KFET_PK, "date_start": row["date"].date(), # Only date, not time "date_end": get_date_end(row["date"].date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 # BDE Membership obj_dict2 = obj_dict.copy() child_dict2 = dict() obj_dict2["pk"] = pk_transaction obj_dict2["amount"] = max(montant - 500, 0) child_dict2["pk"] = pk_transaction bde_dict = { "pk": pk_membership, "user": self.MAP_IDBDE[row["idbde"]], "club": BDE_PK, "date_start": row["date"].date(), # Only date, not time "date_end": get_date_end(row["date"].date()), "fee": max(montant - 500, 0), } pk_membership += 1 # BDE membership Transaction is inserted before the Kfet membershipTransaction bulk_mgr.add( Transaction(**obj_dict2), child_transaction(**child_dict2), Membership(**bde_dict), Membership(**kfet_dict), ) elif ttype == "invitation": child_transaction = GuestTransaction m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: first_name, last_name = m.groups(1), m.groups(2) guest_id = Guest.object.filter(first_name__iexact=first_name, last_name__iexact=last_name).first().pk child_dict["guest_id"] = guest_id else: raise(f"Guest not Found {row['id']} {first_name}, last_name" ) bulk_mgr.add(Transaction(**obj_dict), child_transaction(**child_dict)) pk_transaction += 1 def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["map"]: self.load(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"]) self.import_transaction(cur, kwargs["chunk"])