everything is imported, without server error

management/commands/_import_utils.py

@@ -19,7 +19,7 @@ def timed(method):
         ts = time.time()
         result = method(*args, **kw)
         te = time.time()
-        args[0].print_success(f"{method.__name__} executed ({te-ts:.2f}s)")
+        args[0].print_success(f"\n    {method.__name__} executed ({te-ts:.2f}s)")
         return result
 
     return _timed

management/commands/import_activities.py

@@ -15,6 +15,15 @@ from ._import_utils import ImportCommand, BulkCreateManager, timed
 
 MAP_ACTIVITY = dict()
 
+CLUB_RELOU = [
+    0, # BDE
+    4771, # Kataclist
+    5162, # Assurance BDE ?!
+    5164, #S & L
+    625, #Aspique
+    5154, #Frekens
+]
+
 class Command(ImportCommand):
     """
     Import command for Activities Base Data (Comptes, and Aliases)
@@ -54,11 +63,11 @@ class Command(ImportCommand):
                 "date_start": make_aware(row["debut"]),
                 "date_end": make_aware(row["fin"]),
                 "valid": row["validepar"] is not None,
-                "open": row["ouvert"],  # Should always be False
+                "open": False,
             }
             # WARNING: Fields lieu, liste, listeimprimee are missing
             MAP_ACTIVITY[row["id"]] = pk_activity
-            pk_activity +=1
+            pk_activity += 1
             bulk_mgr.add(Activity(**obj_dict))
         bulk_mgr.done()
 
@@ -70,6 +79,8 @@ class Command(ImportCommand):
         n = cur.rowcount
         for idx, row in enumerate(cur):
             self.update_line(idx, n, f"{row['nom']} {row['prenom']}")
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             obj_dict = {
                 "pk": row["id"],
                 "activity_id": MAP_ACTIVITY[row["activite"]],
@@ -88,11 +99,13 @@ class Command(ImportCommand):
         n = cur.rowcount
         for idx, row in enumerate(cur):
             self.update_line(idx, n, f"{row['idbde']} {row['responsable']}")
+            if row["idbde"] in CLUB_RELOU:
+                row["idbde"] = 3508
             obj_dict = {
                 "activity_id": MAP_ACTIVITY[row["activite"]],
                 "time": make_aware(row["heure_entree"]),
                 "note_id": self.MAP_IDBDE[row["responsable"] if row['est_invite'] else row["idbde"]],
-                "guest_id": self.MAP_IDBDE[row["idbde"]] if row['est_invite'] else None,
+                "guest_id": row["idbde"] if row['est_invite'] else None,
             }
             bulk_mgr.add(Entry(**obj_dict))
         bulk_mgr.done()

management/commands/import_nk15.py

@@ -14,5 +14,5 @@ class Command(BaseCommand):
         subprocess.call("./apps/scripts/shell/tabularasa")
         call_command('import_account', alias=True, chunk=1000, save = "map.json")
         call_command('import_activities', chunk=100, map="map.json")
-        call_command('import_transaction', buttons=True, map="map.json")
+        call_command('import_transaction', chunk=10000, buttons=True, map="map.json")
 #

management/commands/import_transaction.py

@@ -15,7 +15,7 @@ from note.models import (TemplateCategory,
                          RecurrentTransaction,
                          SpecialTransaction
                          )
-from note.models import Note
+from note.models import Note, NoteClub
 from activity.models import Guest, GuestTransaction
 
 from member.models import Membership, MembershipTransaction
@@ -33,10 +33,9 @@ NOTE_SPECIAL_CODE = {
 
 def get_date_end(date_start):
     date_end = copy.deepcopy(date_start)
-    if date_start > 8:
-        date_end.year = date_start + 1
-    date_end.month = 9
-    date_end.day = 30
+    if date_start.month > 8:
+        date_end = date_start.replace(year=date_start.year+1)
+    date_end = date_end.replace(month=9, day=30)
     return date_end
 
 
@@ -50,39 +49,98 @@ class Command(ImportCommand):
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
 
     @timed
-    @transaction.atomic
     def import_buttons(self, cur, chunk_size):
-        categories = dict()
-        buttons = dict()
+        self.categories = dict()
+        self.buttons = dict()
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute("SELECT * FROM boutons;")
         n = cur.rowcount
-        pk_category = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["label"])
-            if row["categorie"] not in categories:
-                bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"]))
-                pk_category += 1
-                categories[row["categorie"]] = pk_category
+            if row["categorie"] not in self.categories:
+                cat = TemplateCategory.objects.create(name=row["categorie"])
+                cat.save()
+                self.categories[row["categorie"]] = cat.pk
             obj_dict = {
                 "pk": row["id"],
                 "name": row["label"],
                 "amount": row["montant"],
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
-                "category_id": categories[row["categorie"]],
+                "category_id": self.categories[row["categorie"]],
                 "display": row["affiche"],
                 "description": row["description"],
             }
-            if row["label"] in buttons:
-                obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}"
+            if row["label"] in self.buttons:
+                obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
             bulk_mgr.add(TransactionTemplate(**obj_dict))
-            buttons[obj_dict["label"]] = row["id"]
+            self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
         bulk_mgr.done()
-        return buttons, categories
+
+    def _basic_transaction(self, row, obj_dict, child_dict):
+        if len(row["description"]) > 255:
+            obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
+        return obj_dict, None, None
+
+    def _template_transaction(self, row, obj_dict, child_dict):
+        if self.categories.get(row["categorie"]):
+            child_dict["category_id"] = self.categories[row["categorie"]]
+        elif "WEI" in row["description"]:
+            return obj_dict, None, None
+        elif self.buttons.get(row["description"]):
+            child_dict["category_id"] = self.buttons[row["description"]][1]
+            child_dict["template_id"] = self.buttons[row["description"]][0]
+        else:
+            return obj_dict, None, None
+        return obj_dict, child_dict, RecurrentTransaction
+
+    def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
+        obj_dict2 = obj_dict.copy()
+        child_dict2 = child_dict.copy()
+        child_dict2["membership_id"] = pk_membership
+
+        return obj_dict2, child_dict2, MembershipTransaction
+
+    def _special_transaction(self, row, obj_dict, child_dict):
+        # Some transaction uses BDE (idbde=0) as source or destination,
+        # lets fix that.
+        field_id = "source_id" if row["type"] == "crédit" else "destination_id"
+        if "espèce" in row["description"]:
+            obj_dict[field_id] = 1
+        elif "carte" in row["description"]:
+            obj_dict[field_id] = 2
+        elif "cheques" in row["description"]:
+            obj_dict[field_id] = 3
+        elif "virement" in row["description"]:
+            obj_dict[field_id] = 4
+        # humans and clubs have always the biggest id
+        actor_pk = max(row["destinataire"], row["emetteur"])
+        actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
+        # custom fields of SpecialTransaction
+        if actor.__class__.__name__ == "NoteUser":
+            child_dict["first_name"] = actor.user.first_name
+            child_dict["last_name"] = actor.user.last_name
+        else:
+            child_dict["first_name"] = actor.club.name
+            child_dict["last_name"] = actor.club.name
+        return obj_dict, child_dict, SpecialTransaction
+
+    def _guest_transaction(self, row, obj_dict, child_dict):
+        # Currently GuestTransaction is related to a Guest.
+        # This is not ideal and should be change to the Entry of this Guest.
+        m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
+        if m:
+            first_name, last_name = m.group(2), m.group(3)
+            guest_id = Guest.objects.filter(first_name__iexact=first_name,
+                                           last_name__iexact=last_name).first().pk
+            child_dict["guest_id"] = guest_id
+        else:
+            raise(f"Guest not Found {row['id']} {first_name}, last_name")
+
+        return obj_dict, child_dict, GuestTransaction
 
     @timed
     @transaction.atomic
-    def import_transaction(self, cur, chunk_size, idmin, buttons, categories):
+    def import_transaction(self, cur, chunk_size, idmin):
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
         cur.execute(
             f"SELECT t.date AS transac_date, t.type, t.emetteur,\
@@ -117,89 +175,68 @@ class Command(ImportCommand):
             # for child transaction Models
             child_dict = {"pk": obj_dict["pk"]}
             ttype = row["type"]
-            if ttype == "don" or ttype == "transfert":
+            if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
+                note = Note.objects.get(pk=obj_dict["source_id"])
+                if isinstance(note, NoteClub):
                     child_transaction = None
-            elif ttype == "bouton":
-                child_transaction = RecurrentTransaction
-                child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"])
-                child_dict["template_id"] = buttons[row["description"]]
-            elif ttype == "crédit" or ttype == "retrait":
-                child_transaction = SpecialTransaction
-                # Some transaction uses BDE (idbde=0) as source or destination,
-                # lets fix that.
-                field_id = "source_id" if ttype == "crédit" else "destination_id"
-                if "espèce" in row["description"]:
-                    obj_dict[field_id] = 1
-                elif "carte" in row["description"]:
-                    obj_dict[field_id] = 2
-                elif "cheques" in row["description"]:
-                    obj_dict[field_id] = 3
-                elif "virement" in row["description"]:
-                    obj_dict[field_id] = 4
-                # humans and clubs have always the biggest id
-                actor_pk = max(row["destinataire"], row["emetteur"])
-                actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
-                # custom fields of SpecialTransaction
-                if actor.__class__.__name__ == "NoteUser":
-                    child_dict["first_name"] = actor.user.first_name
-                    child_dict["last_name"] = actor.user.last_name
                 else:
-                    child_dict["first_name"] = actor.club.name
-                    child_dict["last_name"] = actor.club.name
-            elif ttype == "adhésion" and row["valide"]:
-                child_transaction = MembershipTransaction
-                # Kfet membership
-                montant = row["montant"]
-                obj_dict["amount"] = min(500, montant)
-                child_dict["membership_id"] = pk_membership
-                kfet_dict = {
-                    "pk": pk_membership,
-                    "user": self.MAP_IDBDE[row["idbde"]],
-                    "club": KFET_PK,
-                    "date_start": row["date"].date(),  # Only date, not time
-                    "date_end": get_date_end(row["date"].date()),
-                    "fee": min(500, montant)
-                }
-
-                pk_membership += 1
-                pk_transaction += 1
-                # BDE Membership
-                obj_dict2 = obj_dict.copy()
-                child_dict2 = dict()
-                obj_dict2["pk"] = pk_transaction
-                obj_dict2["amount"] = max(montant - 500, 0)
-                child_dict2["pk"] = pk_transaction
+                    user_id = note.user_id
+                    montant = obj_dict["amount"]
+                    obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
                     bde_dict = {
                         "pk": pk_membership,
-                    "user": self.MAP_IDBDE[row["idbde"]],
-                    "club": BDE_PK,
-                    "date_start": row["date"].date(),  # Only date, not time
-                    "date_end": get_date_end(row["date"].date()),
-                    "fee": max(montant - 500, 0),
+                        "user_id": user_id,
+                        "club_id": KFET_PK,
+                        "date_start": date.date(),  # Only date, not time
+                        "date_end": get_date_end(date.date()),
+                        "fee": min(500, montant)
                     }
                     pk_membership += 1
+                    pk_transaction += 1
+                    obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    # Kfet membership
+                    # BDE Membership
+                    obj_dict["pk"] = pk_transaction
+                    child_dict["pk"] = pk_transaction
+                    kfet_dict = {
+                        "pk": pk_membership,
+                        "user_id": user_id,
+                        "club_id": BDE_PK,
+                        "date_start": date.date(),  # Only date, not time
+                        "date_end": get_date_end(date.date()),
+                        "fee": max(montant - 500, 0),
+                    }
+                    obj_dict0["amount"] = bde_dict["fee"]
+                    obj_dict["amount"] = kfet_dict["fee"]
                     # BDE membership Transaction is inserted before the Kfet membershipTransaction
+                    pk_membership += 1
+                    pk_transaction += 1
                     bulk_mgr.add(
-                    Transaction(**obj_dict2),
-                    child_transaction(**child_dict2),
+                        Transaction(**obj_dict0),
+                        child_transaction(**child_dict0),
+                        Transaction(**obj_dict),
+                        child_transaction(**child_dict),
                         Membership(**bde_dict),
                         Membership(**kfet_dict),
                     )
+                    continue
+            elif ttype == "bouton":
+                obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
+            elif ttype == "crédit" or ttype == "retrait":
+                obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
             elif ttype == "invitation":
-                child_transaction = GuestTransaction
-                m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
-                if m:
-                    first_name, last_name = m.groups(1), m.groups(2)
-                    guest_id = Guest.object.filter(first_name__iexact=first_name,
-                                                   last_name__iexact=last_name).first().pk
-                    child_dict["guest_id"] = guest_id
+                obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
+            if ttype == "don" or ttype == "transfert":
+                obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
             else:
-                    raise(f"Guest not Found {row['id']} {first_name}, last_name" )
-
-            bulk_mgr.add(Transaction(**obj_dict),
-                         child_transaction(**child_dict))
+                child_transaction = None
+                # create base transaction object and typed one
+            bulk_mgr.add(Transaction(**obj_dict))
+            if child_transaction is not None:
+                bulk_mgr.add(child_transaction(**child_dict))
             pk_transaction += 1
-           
+        bulk_mgr.done()
+    @timed
     def handle(self, *args, **kwargs):
         # default args, provided by ImportCommand.
         nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
@@ -208,8 +245,6 @@ class Command(ImportCommand):
         cur = conn.cursor(cursor_factory=pge.DictCursor)
 
         if kwargs["map"]:
-            self.load(kwargs["map"])
-           
+            self.load_map(kwargs["map"])
         self.import_buttons(cur, kwargs["chunk"])
-       
-        self.import_transaction(cur, kwargs["chunk"])
+        self.import_transaction(cur, kwargs["chunk"], 0)

shell/tabularasa

@@ -1,6 +1,7 @@
 #!/usr/bin/sh
 sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
 echo 'reset db';
-source "env/bin/activate"
+find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete
+./manage.py makemigrations
 ./manage.py migrate
 ./manage.py loaddata initial