From c19a0582bd73f2e0bd14d2e4524d127b680c9e79 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 26 Jul 2020 23:59:04 +0200 Subject: [PATCH] Fix remittance import --- management/commands/import_transaction.py | 59 ++++++++++++++--------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py index 7d40658..8911965 100644 --- a/management/commands/import_transaction.py +++ b/management/commands/import_transaction.py @@ -67,6 +67,8 @@ class Command(ImportCommand): def add_arguments(self, parser): parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") + parser.add_argument('-n', '--nosave', action='store_true', default=True, help="Scan only transactions, " + "don't save them") @timed def import_buttons(self, cur, chunk_size, import_buttons): @@ -169,7 +171,7 @@ class Command(ImportCommand): @timed @transaction.atomic - def import_transaction(self, cur, chunk_size, idmin): + def import_transaction(self, cur, chunk_size, idmin, save=True): bulk_mgr = BulkCreateManager(chunk_size=chunk_size) cur.execute( f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\ @@ -184,10 +186,19 @@ class Command(ImportCommand): pk_membership = 1 pk_transaction = 1 for idx, row in enumerate(cur): - self.update_line(idx, n, row["description"]) + if save or idx % chunk_size == 0: + self.update_line(idx, n, row["description"]) MAP_TRANSACTION[row["id"]] = pk_transaction + if not save: + pk_transaction += 1 + if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"): + note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]]) + if not isinstance(note, NoteClub): + pk_transaction += 1 + continue + try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): @@ -217,7 +228,7 @@ class Command(ImportCommand): if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): - child_transaction = None # don't bother register clubs + child_transaction = None # don't bother register clubs else: user_id = note.user_id montant = obj_dict["amount"] @@ -280,6 +291,20 @@ class Command(ImportCommand): pk_transaction += 1 bulk_mgr.done() + @timed + def set_roles(self): + bulk_mgr = BulkCreateManager(chunk_size=10000) + bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True) + kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True) + n = len(bde_membership_ids) + for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)): + self.update_line(idx, n, str(idx)) + bulk_mgr.add( + Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK), + Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK), + ) + bulk_mgr.done() + @timed @transaction.atomic def import_remittances(self, cur, chunk_size): @@ -295,7 +320,7 @@ class Command(ImportCommand): remittance_dict = { "pk": pk_remittance, - "date": make_aware(row["date"][:19]), + "date": make_aware(row["date"]), "remittance_type_id": 1, # Only Bank checks are supported in NK15 "comment": row["commentaire"], "closed": row["close"], @@ -318,7 +343,7 @@ class Command(ImportCommand): if not row["idremise"]: continue - tr = SpecialTransactionProxy.objects.get(transaction__id=MAP_TRANSACTION[row["idtransaction"]]) + tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0] tr.remittance_id = MAP_REMITTANCE[row["idremise"]] tr.save() @@ -326,21 +351,10 @@ class Command(ImportCommand): tr.last_name = row["nom"] tr.first_name = row["prenom"] tr.bank = row["banque"] - tr.save() - - @timed - def set_roles(self): - bulk_mgr = BulkCreateManager(chunk_size=10000) - bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True) - kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True) - n = len(bde_membership_ids) - for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)): - self.update_line(idx, n, str(idx)) - bulk_mgr.add( - Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK), - Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK), - ) - bulk_mgr.done() + try: + tr.save() + except: + print("Failed to save row: " + str(row)) @timed def handle(self, *args, **kwargs): @@ -353,7 +367,8 @@ class Command(ImportCommand): if kwargs["map"]: self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) - self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"]) - self.set_roles() + self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"]) + if not kwargs["nosave"]: + self.set_roles() self.import_remittances(cur, kwargs["chunk"]) self.import_checks(cur)