From 0ba656d5e016581ad7d635729cd6514ebd387ad8 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Tue, 26 May 2020 23:54:24 +0200 Subject: [PATCH] everything is imported, without server error --- management/commands/import_activities.py | 17 +- management/commands/import_transaction.py | 239 +++++++++++++--------- 2 files changed, 152 insertions(+), 104 deletions(-) diff --git a/management/commands/import_activities.py b/management/commands/import_activities.py index e4fd7f8..55b79fc 100644 --- a/management/commands/import_activities.py +++ b/management/commands/import_activities.py @@ -15,6 +15,15 @@ from ._import_utils import ImportCommand, BulkCreateManager, timed MAP_ACTIVITY = dict() +CLUB_RELOU = [ + 0, # BDE + 4771, # Kataclist + 5162, # Assurance BDE ?! + 5164, #S & L + 625, #Aspique + 5154, #Frekens +] + class Command(ImportCommand): """ Import command for Activities Base Data (Comptes, and Aliases) @@ -54,7 +63,7 @@ class Command(ImportCommand): "date_start": make_aware(row["debut"]), "date_end": make_aware(row["fin"]), "valid": row["validepar"] is not None, - "open": row["ouvert"], # Should always be False + "open": False, } # WARNING: Fields lieu, liste, listeimprimee are missing MAP_ACTIVITY[row["id"]] = pk_activity @@ -70,6 +79,8 @@ class Command(ImportCommand): n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, f"{row['nom']} {row['prenom']}") + if row["responsable"] in CLUB_RELOU: + row["responsable"] = 3508 obj_dict = { "pk": row["id"], "activity_id": MAP_ACTIVITY[row["activite"]], @@ -88,11 +99,13 @@ class Command(ImportCommand): n = cur.rowcount for idx, row in enumerate(cur): self.update_line(idx, n, f"{row['idbde']} {row['responsable']}") + if row["idbde"] in CLUB_RELOU: + row["idbde"] = 3508 obj_dict = { "activity_id": MAP_ACTIVITY[row["activite"]], "time": make_aware(row["heure_entree"]), "note_id": self.MAP_IDBDE[row["responsable"] if row['est_invite'] else row["idbde"]], - "guest_id": self.MAP_IDBDE[row["idbde"]] if row['est_invite'] else None, + "guest_id": row["idbde"] if row['est_invite'] else None, } bulk_mgr.add(Entry(**obj_dict)) bulk_mgr.done() diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py index 5d6cbca..78c97db 100644 --- a/management/commands/import_transaction.py +++ b/management/commands/import_transaction.py @@ -15,7 +15,7 @@ from note.models import (TemplateCategory, RecurrentTransaction, SpecialTransaction ) -from note.models import Note +from note.models import Note, NoteClub from activity.models import Guest, GuestTransaction from member.models import Membership, MembershipTransaction @@ -33,10 +33,9 @@ NOTE_SPECIAL_CODE = { def get_date_end(date_start): date_end = copy.deepcopy(date_start) - if date_start > 8: - date_end.year = date_start + 1 - date_end.month = 9 - date_end.day = 30 + if date_start.month > 8: + date_end = date_start.replace(year=date_start.year+1) + date_end = date_end.replace(month=9, day=30) return date_end @@ -50,39 +49,98 @@ class Command(ImportCommand): parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") @timed - @transaction.atomic def import_buttons(self, cur, chunk_size): - categories = dict() - buttons = dict() + self.categories = dict() + self.buttons = dict() bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute("SELECT * FROM boutons;") n = cur.rowcount - pk_category = 1 for idx, row in enumerate(cur): self.update_line(idx, n, row["label"]) - if row["categorie"] not in categories: - bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"])) - pk_category += 1 - categories[row["categorie"]] = pk_category + if row["categorie"] not in self.categories: + cat = TemplateCategory.objects.create(name=row["categorie"]) + cat.save() + self.categories[row["categorie"]] = cat.pk obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": self.MAP_IDBDE[row["destinataire"]], - "category_id": categories[row["categorie"]], + "category_id": self.categories[row["categorie"]], "display": row["affiche"], "description": row["description"], } - if row["label"] in buttons: - obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}" + if row["label"] in self.buttons: + obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}" bulk_mgr.add(TransactionTemplate(**obj_dict)) - buttons[obj_dict["label"]] = row["id"] + self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]]) bulk_mgr.done() - return buttons, categories + + def _basic_transaction(self, row, obj_dict, child_dict): + if len(row["description"]) > 255: + obj_dict["reason"] = obj_dict["reason"][:250]+"...)" + return obj_dict, None, None + + def _template_transaction(self, row, obj_dict, child_dict): + if self.categories.get(row["categorie"]): + child_dict["category_id"] = self.categories[row["categorie"]] + elif "WEI" in row["description"]: + return obj_dict, None, None + elif self.buttons.get(row["description"]): + child_dict["category_id"] = self.buttons[row["description"]][1] + child_dict["template_id"] = self.buttons[row["description"]][0] + else: + return obj_dict, None, None + return obj_dict, child_dict, RecurrentTransaction + + def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): + obj_dict2 = obj_dict.copy() + child_dict2 = child_dict.copy() + child_dict2["membership_id"] = pk_membership + + return obj_dict2, child_dict2, MembershipTransaction + + def _special_transaction(self, row, obj_dict, child_dict): + # Some transaction uses BDE (idbde=0) as source or destination, + # lets fix that. + field_id = "source_id" if row["type"] == "crédit" else "destination_id" + if "espèce" in row["description"]: + obj_dict[field_id] = 1 + elif "carte" in row["description"]: + obj_dict[field_id] = 2 + elif "cheques" in row["description"]: + obj_dict[field_id] = 3 + elif "virement" in row["description"]: + obj_dict[field_id] = 4 + # humans and clubs have always the biggest id + actor_pk = max(row["destinataire"], row["emetteur"]) + actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) + # custom fields of SpecialTransaction + if actor.__class__.__name__ == "NoteUser": + child_dict["first_name"] = actor.user.first_name + child_dict["last_name"] = actor.user.last_name + else: + child_dict["first_name"] = actor.club.name + child_dict["last_name"] = actor.club.name + return obj_dict, child_dict, SpecialTransaction + + def _guest_transaction(self, row, obj_dict, child_dict): + # Currently GuestTransaction is related to a Guest. + # This is not ideal and should be change to the Entry of this Guest. + m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) + if m: + first_name, last_name = m.group(2), m.group(3) + guest_id = Guest.objects.filter(first_name__iexact=first_name, + last_name__iexact=last_name).first().pk + child_dict["guest_id"] = guest_id + else: + raise(f"Guest not Found {row['id']} {first_name}, last_name") + + return obj_dict, child_dict, GuestTransaction @timed @transaction.atomic - def import_transaction(self, cur, chunk_size, idmin, buttons, categories): + def import_transaction(self, cur, chunk_size, idmin): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.date AS transac_date, t.type, t.emetteur,\ @@ -117,89 +175,68 @@ class Command(ImportCommand): # for child transaction Models child_dict = {"pk": obj_dict["pk"]} ttype = row["type"] - if ttype == "don" or ttype == "transfert": - child_transaction = None + if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): + note = Note.objects.get(pk=obj_dict["source_id"]) + if isinstance(note, NoteClub): + child_transaction = None + else: + user_id = note.user_id + montant = obj_dict["amount"] + obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + bde_dict = { + "pk": pk_membership, + "user_id": user_id, + "club_id": KFET_PK, + "date_start": date.date(), # Only date, not time + "date_end": get_date_end(date.date()), + "fee": min(500, montant) + } + pk_membership += 1 + pk_transaction += 1 + obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + # Kfet membership + # BDE Membership + obj_dict["pk"] = pk_transaction + child_dict["pk"] = pk_transaction + kfet_dict = { + "pk": pk_membership, + "user_id": user_id, + "club_id": BDE_PK, + "date_start": date.date(), # Only date, not time + "date_end": get_date_end(date.date()), + "fee": max(montant - 500, 0), + } + obj_dict0["amount"] = bde_dict["fee"] + obj_dict["amount"] = kfet_dict["fee"] + # BDE membership Transaction is inserted before the Kfet membershipTransaction + pk_membership += 1 + pk_transaction += 1 + bulk_mgr.add( + Transaction(**obj_dict0), + child_transaction(**child_dict0), + Transaction(**obj_dict), + child_transaction(**child_dict), + Membership(**bde_dict), + Membership(**kfet_dict), + ) + continue elif ttype == "bouton": - child_transaction = RecurrentTransaction - child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"]) - child_dict["template_id"] = buttons[row["description"]] + obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict) elif ttype == "crédit" or ttype == "retrait": - child_transaction = SpecialTransaction - # Some transaction uses BDE (idbde=0) as source or destination, - # lets fix that. - field_id = "source_id" if ttype == "crédit" else "destination_id" - if "espèce" in row["description"]: - obj_dict[field_id] = 1 - elif "carte" in row["description"]: - obj_dict[field_id] = 2 - elif "cheques" in row["description"]: - obj_dict[field_id] = 3 - elif "virement" in row["description"]: - obj_dict[field_id] = 4 - # humans and clubs have always the biggest id - actor_pk = max(row["destinataire"], row["emetteur"]) - actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) - # custom fields of SpecialTransaction - if actor.__class__.__name__ == "NoteUser": - child_dict["first_name"] = actor.user.first_name - child_dict["last_name"] = actor.user.last_name - else: - child_dict["first_name"] = actor.club.name - child_dict["last_name"] = actor.club.name - elif ttype == "adhésion" and row["valide"]: - child_transaction = MembershipTransaction - # Kfet membership - montant = row["montant"] - obj_dict["amount"] = min(500, montant) - child_dict["membership_id"] = pk_membership - kfet_dict = { - "pk": pk_membership, - "user": self.MAP_IDBDE[row["idbde"]], - "club": KFET_PK, - "date_start": row["date"].date(), # Only date, not time - "date_end": get_date_end(row["date"].date()), - "fee": min(500, montant) - } - - pk_membership += 1 - pk_transaction += 1 - # BDE Membership - obj_dict2 = obj_dict.copy() - child_dict2 = dict() - obj_dict2["pk"] = pk_transaction - obj_dict2["amount"] = max(montant - 500, 0) - child_dict2["pk"] = pk_transaction - bde_dict = { - "pk": pk_membership, - "user": self.MAP_IDBDE[row["idbde"]], - "club": BDE_PK, - "date_start": row["date"].date(), # Only date, not time - "date_end": get_date_end(row["date"].date()), - "fee": max(montant - 500, 0), - } - pk_membership += 1 - # BDE membership Transaction is inserted before the Kfet membershipTransaction - bulk_mgr.add( - Transaction(**obj_dict2), - child_transaction(**child_dict2), - Membership(**bde_dict), - Membership(**kfet_dict), - ) + obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": - child_transaction = GuestTransaction - m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) - if m: - first_name, last_name = m.groups(1), m.groups(2) - guest_id = Guest.object.filter(first_name__iexact=first_name, - last_name__iexact=last_name).first().pk - child_dict["guest_id"] = guest_id - else: - raise(f"Guest not Found {row['id']} {first_name}, last_name" ) - - bulk_mgr.add(Transaction(**obj_dict), - child_transaction(**child_dict)) + obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) + if ttype == "don" or ttype == "transfert": + obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) + else: + child_transaction = None + # create base transaction object and typed one + bulk_mgr.add(Transaction(**obj_dict)) + if child_transaction is not None: + bulk_mgr.add(child_transaction(**child_dict)) pk_transaction += 1 - + bulk_mgr.done() + @timed def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] @@ -208,8 +245,6 @@ class Command(ImportCommand): cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["map"]: - self.load(kwargs["map"]) - + self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"]) - - self.import_transaction(cur, kwargs["chunk"]) + self.import_transaction(cur, kwargs["chunk"], 0)