Better import

management/commands/_import_utils.py

@@ -44,7 +44,7 @@ class ImportCommand(BaseCommand):
         n = str(n)
         total = str(total)
         n.rjust(len(total))
-        print(f"\r ({n}/{total}) {content:10.10}", end="")
+        print(f"\r ({n}/{total}) {content:16.16}", end="")
 
     def create_parser(self, prog_name, subcommand, **kwargs):
         parser = super().create_parser(prog_name, subcommand, **kwargs)

management/commands/import_nk15.py

@@ -30,4 +30,6 @@ class Command(ImportCommand):
         kwargs["map"] = "map.json"
         kwargs["buttons"] = True
         call_command('import_transaction', **kwargs)
-#
+
+        call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant')
+        call_command('syncsql')

management/commands/import_transaction.py

@@ -8,12 +8,14 @@ import copy
 
 from django.utils.timezone import make_aware
 from django.db import transaction
+from django.contrib.contenttypes.models import ContentType
 
 from note.models import (TemplateCategory,
                          TransactionTemplate,
                          Transaction,
                          RecurrentTransaction,
-                         SpecialTransaction
+                         SpecialTransaction,
+                         MembershipTransaction,
                          )
 from note.models import Note, NoteClub
 from activity.models import Guest, GuestTransaction
@@ -21,15 +23,27 @@ from activity.models import Guest, GuestTransaction
 from member.models import Membership, MembershipTransaction, Role
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
+# from member/fixtures/initial
 BDE_PK = 1
 KFET_PK = 2
+
+# from note/fixtures/initial
 NOTE_SPECIAL_CODE = {
     "espèce": 1,
     "carte": 2,
     "chèque": 3,
     "virement": 4,
 }
+# from permission/fixtures/initial
+BDE_ROLE_PK = 1
+KFET_ROLE_PK = 2
 
+CT = {
+    "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
+    "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
+    "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
+    "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
+}
 
 def get_date_end(date_start):
     date_end = copy.deepcopy(date_start)
@@ -79,22 +93,24 @@ class Command(ImportCommand):
 
     def _basic_transaction(self, row, obj_dict, child_dict):
         if len(row["description"]) > 255:
-            obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
+            obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
         return obj_dict, None, None
 
     def _template_transaction(self, row, obj_dict, child_dict):
-        if self.categories.get(row["categorie"]):
+        if self.buttons.get(row["description"]):
-            child_dict["category_id"] = self.categories[row["categorie"]]
-        elif "WEI" in row["description"]:
-            return obj_dict, None, None
-        elif self.buttons.get(row["description"]):
             child_dict["category_id"] = self.buttons[row["description"]][1]
             child_dict["template_id"] = self.buttons[row["description"]][0]
+        # elif self.categories.get(row["categorie"]):
+        #     child_dict["category_id"] = self.categories[row["categorie"]]
+        elif "WEI" in row["description"]:
+            return obj_dict, None, None
         else:
             return obj_dict, None, None
+        obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
         return obj_dict, child_dict, RecurrentTransaction
 
     def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
+        obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
         obj_dict2 = obj_dict.copy()
         child_dict2 = child_dict.copy()
         child_dict2["membership_id"] = pk_membership
@@ -104,6 +120,7 @@ class Command(ImportCommand):
     def _special_transaction(self, row, obj_dict, child_dict):
         # Some transaction uses BDE (idbde=0) as source or destination,
         # lets fix that.
+        obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
         field_id = "source_id" if row["type"] == "crédit" else "destination_id"
         if "espèce" in row["description"]:
             obj_dict[field_id] = 1
@@ -128,6 +145,7 @@ class Command(ImportCommand):
     def _guest_transaction(self, row, obj_dict, child_dict):
         # Currently GuestTransaction is related to a Guest.
         # This is not ideal and should be change to the Entry of this Guest.
+        obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
         m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
         if m:
             first_name, last_name = m.group(2), m.group(3)
@@ -171,35 +189,42 @@ class Command(ImportCommand):
             obj_dict = {
                 "pk": pk_transaction,
                 "destination_id": self.MAP_IDBDE[row["destinataire"]],
+                "polymorphic_ctype": None,
                 "source_id": self.MAP_IDBDE[row["emetteur"]],
-                "created_at": date,
                 "amount": row["montant"],
+                "created_at": date,
+                "destination_alias": "",
+                "invalidity_reason": None,
                 "quantity": row["quantite"],
                 "reason": row["description"],
+                "source_alias": "",
                 "valid": row["valide"],
             }
             # for child transaction Models
-            child_dict = {"pk": obj_dict["pk"]}
+            child_dict = {"pk": pk_transaction}
             ttype = row["type"]
+            # Membership transaction detection and import
             if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
                 note = Note.objects.get(pk=obj_dict["source_id"])
                 if isinstance(note, NoteClub):
-                    child_transaction = None
+                    child_transaction = None # don't bother register clubs
                 else:
                     user_id = note.user_id
                     montant = obj_dict["amount"]
-                    obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    (obj_dict0,
+                     child_dict0,
+                     child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                     bde_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": KFET_PK,
+                        "club_id": BDE_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": min(500, montant)
                     }
                     pk_membership += 1
                     pk_transaction += 1
-                    obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
+                    obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
                     # Kfet membership
                     # BDE Membership
                     obj_dict["pk"] = pk_transaction
@@ -207,7 +232,7 @@ class Command(ImportCommand):
                     kfet_dict = {
                         "pk": pk_membership,
                         "user_id": user_id,
-                        "club_id": BDE_PK,
+                        "club_id": KFET_PK,
                         "date_start": date.date(),  # Only date, not time
                         "date_end": get_date_end(date.date()),
                         "fee": max(montant - 500, 0),
@@ -218,12 +243,12 @@ class Command(ImportCommand):
                     pk_membership += 1
                     pk_transaction += 1
                     bulk_mgr.add(
+                        Membership(**bde_dict),
+                        Membership(**kfet_dict),
                         Transaction(**obj_dict0),
                         child_transaction(**child_dict0),
                         Transaction(**obj_dict),
                         child_transaction(**child_dict),
-                        Membership(**bde_dict),
-                        Membership(**kfet_dict),
                     )
                     continue
             elif ttype == "bouton":
@@ -232,7 +257,7 @@ class Command(ImportCommand):
                 obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
             elif ttype == "invitation":
                 obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
-            if ttype == "don" or ttype == "transfert":
+            elif ttype == "don" or ttype == "transfert":
                 obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
             else:
                 child_transaction = None
@@ -243,18 +268,15 @@ class Command(ImportCommand):
             pk_transaction += 1
         bulk_mgr.done()
 
-    @timed
+    def set_roles(self):
-    def adjust_roles(self):
+        bulk_mgr = BulkCreateManager(chunk_size=10000)
-        bdeRole = Role.objects.get(name="Adhérent BDE")
+        membership_ids = Membership.objects.values_list('id',flat=True)
-        kfetRole = Role.objects.get(name="Adhérent Kfet")
+        for m_id in membership_ids:
-        memberships = Membership.objects.all()
+            bulk_mgr.add(
-        n = len(memberships)
+                Membership.roles.through(membership_id=m_id,role_id=BDE_ROLE_PK),
-        for idx, membership in enumerate(memberships):
+                Membership.roles.through(membership_id=m_id,role_id=KFET_ROLE_PK),
-            self.update_line(idx, n, membership.user.username)
+            )
-            if membership.club.name == "BDE":
+            bulk_mgr.done()
-                membership.roles.set([bdeRole])
-            elif membership.club.name == "Kfet":
-                membership.roles.set([kfetRole])
 
     @timed
     def handle(self, *args, **kwargs):
@@ -268,4 +290,4 @@ class Command(ImportCommand):
             self.load_map(kwargs["map"])
         self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
         self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"])
-        self.adjust_roles()
+        self.set_roles()

management/commands/syncsql.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+
+from django.core.management.base import BaseCommand
+from django.apps import apps
+from django.db import connection
+
+
+from polymorphic.models import PolymorphicModel
+
+NO_SEQ = [
+    "Session",
+    "Token",
+    "WEIRole", # dirty fix
+]
+
+class Command(BaseCommand):
+    """
+    Command to synchronise primary sequence of postgres after bulk insert of django.
+    """
+
+    def add_arguments(self,parser):
+        parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
+        return parser
+
+    def handle(self, *args, **kwargs):
+        app_list = kwargs["apps"]
+        if len(app_list):
+            model_classes = list()
+            for app in app_list:
+                model_classes += apps.get_app_config(app).get_models()
+        else:
+            # no app specified, sync everything
+            model_classes = apps.get_models(include_auto_created=True)
+
+        db_names = [ m._meta.db_table for m in model_classes if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count()>1]
+        com = "BEGIN;\n"
+        for db_name in db_names:
+            com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "{db_name}";\n'
+        com += "COMMIT;"
+        print(com)
+        cur = connection.cursor()
+        cur.execute(com)
+        cur.close()

shell/tabularasa

@@ -1,4 +1,6 @@
 #!/usr/bin/sh
+sudo service postgresql stop
+sudo service postgresql start
 sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
 echo 'reset db';
 find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete