Better import

management/commands/_import_utils.py

@@ -44,14 +44,14 @@ class ImportCommand(BaseCommand):
         n = str(n)
         total = str(total)
         n.rjust(len(total))
-        print(f"\r ({n}/{total}) {content:10.10}", end="")
+        print(f"\r ({n}/{total}) {content:16.16}", end="")
 
     def create_parser(self, prog_name, subcommand, **kwargs):
         parser = super().create_parser(prog_name, subcommand, **kwargs)
         parser.add_argument('--nk15db', action='store', default='nk15', help='NK15 database name')
         parser.add_argument('--nk15user', action='store', default='nk15_user', help='NK15 database owner')
-        parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
+        parser.add_argument('-s', '--save', default='map.json', action='store', help="save mapping of idbde")
-        parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
+        parser.add_argument('-m', '--map', default='map.json', action='store', help="import mapping of idbde")
         parser.add_argument('-c', '--chunk', type=int, default=100, help="chunk size for bulk_create")
         return parser
 

management/commands/import_activities.py

@@ -10,7 +10,7 @@ from django.db import transaction
 
 from activity.models import ActivityType, Activity, Guest, Entry
 from member.models import Club
-from note.models import Note
+from note.models import Note, NoteUser
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
 MAP_ACTIVITY = dict()
@@ -22,6 +22,10 @@ CLUB_RELOU = [
     5164, # S & L
     625,  # Aspique
     5154, # Frekens
+    3944, # DiskJok[ENS]
+    5153, # Monopo[list]
+    2351, # JdRM
+    2365, # Pot Vieux
 ]
 
 class Command(ImportCommand):
@@ -40,11 +44,13 @@ class Command(ImportCommand):
         pk_activity = 1
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["titre"])
+            if row["responsable"] in CLUB_RELOU:
+                row["responsable"] = 3508
             note = self.MAP_IDBDE[row["responsable"]]
             if note == 6244:
                  # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
                 note = Note.objects.get(pk=self.MAP_IDBDE[6524])
-                note = note.user_id
+                note = note.id
             organizer = Club.objects.filter(name=row["signature"])
             if organizer.exists():
                 # Try to find the club that organizes the activity.
@@ -57,7 +63,7 @@ class Command(ImportCommand):
                 "name": row["titre"],
                 "description": row["description"],
                 "activity_type_id": activity_type_id,  # By default Pot
-                "creater_id": note,
+                "creater_id": NoteUser.objects.get(pk=note).user.id,
                 "organizer_id": organizer.pk,
                 "attendees_club_id": kfet.pk,  # Maybe fix manually
                 "date_start": make_aware(row["debut"]),

management/commands/import_nk15.py

@@ -4,7 +4,9 @@ import subprocess
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
-class Command(BaseCommand):
+from ._import_utils import ImportCommand
+
+class Command(ImportCommand):
     """
     Command for importing the database of NK15.
     Need to be run by a user with a registered role in postgres for the database nk15. 
@@ -12,8 +14,22 @@ class Command(BaseCommand):
 
     def handle(self, *args, **kwargs):
         subprocess.call("./apps/scripts/shell/tabularasa")
-        call_command('import_account', alias=True, chunk=5000, save = "map.json")
+
-        call_command('import_activities', chunk=5000, map="map.json")
+        kwargs["alias"] = True
-        call_command('import_transaction', chunk=5000, buttons=True, map="map.json")
+        kwargs["chunk"] = 1000
+        kwargs["save"] = "map.json"
+        call_command('import_account', **kwargs)
+
+        del kwargs["alias"]
+        del kwargs["save"]
+        kwargs["chunk"] = 100
+        kwargs["map"] = "map.json"
+        call_command('import_activities', **kwargs)
+
+        kwargs["chunk"] = 10000
+        kwargs["map"] = "map.json"
+        kwargs["buttons"] = True
+        call_command('import_transaction', **kwargs)
+
         call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant')
         call_command('syncsql')

management/commands/import_transaction.py

@@ -20,7 +20,7 @@ from note.models import (TemplateCategory,
 from note.models import Note, NoteClub
 from activity.models import Guest, GuestTransaction
 
-from member.models import Membership
+from member.models import Membership, MembershipTransaction, Role
 from ._import_utils import ImportCommand, BulkCreateManager, timed
 
 # from member/fixtures/initial
@@ -63,7 +63,7 @@ class Command(ImportCommand):
         parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
 
     @timed
-    def import_buttons(self, cur, chunk_size):
+    def import_buttons(self, cur, chunk_size, import_buttons):
         self.categories = dict()
         self.buttons = dict()
         bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
@@ -72,7 +72,7 @@ class Command(ImportCommand):
         for idx, row in enumerate(cur):
             self.update_line(idx, n, row["label"])
             if row["categorie"] not in self.categories:
-                cat = TemplateCategory.objects.create(name=row["categorie"])
+                cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
                 cat.save()
                 self.categories[row["categorie"]] = cat.pk
             obj_dict = {
@@ -86,6 +86,7 @@ class Command(ImportCommand):
             }
             if row["label"] in self.buttons:
                 obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
+            if import_buttons:
                 bulk_mgr.add(TransactionTemplate(**obj_dict))
             self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
         bulk_mgr.done()
@@ -148,6 +149,8 @@ class Command(ImportCommand):
         m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
         if m:
             first_name, last_name = m.group(2), m.group(3)
+            if first_name == "Marion" and last_name == "Bizu Pose":
+                first_name, last_name = "Marion Bizu", "Pose"
             guest_id = Guest.objects.filter(first_name__iexact=first_name,
                                             last_name__iexact=last_name).first().pk
             child_dict["guest_id"] = guest_id
@@ -179,6 +182,9 @@ class Command(ImportCommand):
             except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
                 date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))
 
+            if len(row["description"]) > 255:
+                row["description"] = row["description"][:252] + "..."
+
             # standart transaction object
             obj_dict = {
                 "pk": pk_transaction,
@@ -282,6 +288,6 @@ class Command(ImportCommand):
 
         if kwargs["map"]:
             self.load_map(kwargs["map"])
-        self.import_buttons(cur, kwargs["chunk"])
+        self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
-        self.import_transaction(cur, kwargs["chunk"], 0)
+        self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"])
         self.set_roles()