Fix remittance import

management/commands/import_transaction.py

@@ -67,6 +67,8 @@ class Command(ImportCommand):
     def add_arguments(self, parser):
         parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
+        parser.add_argument('-n', '--nosave', action='store_true', default=True, help="Scan only transactions, "
+                                                                                      "don't save them")
 
     @timed
     def import_buttons(self, cur, chunk_size, import_buttons):
@@ -169,7 +171,7 @@ class Command(ImportCommand):
 
     @timed
     @transaction.atomic
-    def import_transaction(self, cur, chunk_size, idmin):
+    def import_transaction(self, cur, chunk_size, idmin, save=True):
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute(
             f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
@@ -184,10 +186,19 @@ class Command(ImportCommand):
         pk_membership = 1
         pk_transaction = 1
         for idx, row in enumerate(cur):
-            self.update_line(idx, n, row["description"])
+            if save or idx % chunk_size == 0:
+                self.update_line(idx, n, row["description"])
 
             MAP_TRANSACTION[row["id"]] = pk_transaction
 
+            if not save:
+                pk_transaction += 1
+                if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
+                    note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
+                    if not isinstance(note, NoteClub):
+                        pk_transaction += 1
+                continue
+
             try:
                 date = make_aware(row["transac_date"])
             except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
@@ -217,7 +228,7 @@ class Command(ImportCommand):
             if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                 note = Note.objects.get(pk=obj_dict["source_id"])
                 if isinstance(note, NoteClub):
-                    child_transaction = None # don't bother register clubs
+                    child_transaction = None    # don't bother register clubs
                 else:
                     user_id = note.user_id
                     montant = obj_dict["amount"]
@@ -280,6 +291,20 @@ class Command(ImportCommand):
             pk_transaction += 1
         bulk_mgr.done()
 
+    @timed
+    def set_roles(self):
+        bulk_mgr = BulkCreateManager(chunk_size=10000)
+        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
+        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
+        n = len(bde_membership_ids)
+        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
+            self.update_line(idx, n, str(idx))
+            bulk_mgr.add(
+                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
+                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
+            )
+            bulk_mgr.done()
+
     @timed
     @transaction.atomic
     def import_remittances(self, cur, chunk_size):
@@ -295,7 +320,7 @@ class Command(ImportCommand):
 
             remittance_dict = {
                 "pk": pk_remittance,
-                "date": make_aware(row["date"][:19]),
+                "date": make_aware(row["date"]),
                 "remittance_type_id": 1,    # Only Bank checks are supported in NK15
                 "comment": row["commentaire"],
                 "closed": row["close"],
@@ -318,7 +343,7 @@ class Command(ImportCommand):
             if not row["idremise"]:
                 continue
 
-            tr = SpecialTransactionProxy.objects.get(transaction__id=MAP_TRANSACTION[row["idtransaction"]])
+            tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
             tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
             tr.save()
 
@@ -326,21 +351,10 @@ class Command(ImportCommand):
             tr.last_name = row["nom"]
             tr.first_name = row["prenom"]
             tr.bank = row["banque"]
-            tr.save()
-
-    @timed
-    def set_roles(self):
-        bulk_mgr = BulkCreateManager(chunk_size=10000)
-        bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
-        kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
-        n = len(bde_membership_ids)
-        for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
-            self.update_line(idx, n, str(idx))
-            bulk_mgr.add(
-                Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
-                Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
-            )
-            bulk_mgr.done()
+            try:
+                tr.save()
+            except:
+                print("Failed to save row: " + str(row))
 
     @timed
     def handle(self, *args, **kwargs):
@@ -353,7 +367,8 @@ class Command(ImportCommand):
         if kwargs["map"]:
             self.load_map(kwargs["map"])
         self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
-        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"])
-        self.set_roles()
+        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
+        if not kwargs["nosave"]:
+            self.set_roles()
         self.import_remittances(cur, kwargs["chunk"])
         self.import_checks(cur)