Import activities, guests and guest transactions

management/commands/import_nk15.py

@@ -2,25 +2,24 @@
 
 from django.core.management.base import BaseCommand
 from django.core.management import call_command
-from django.utils import timezone
-import psycopg2 as  pg
+import psycopg2 as pg
 import psycopg2.extras as pge
 from django.db import transaction
 
 import json
 import datetime
-import collections
+import re
 
 from django.core.exceptions import ValidationError
 from django.utils.timezone import make_aware
-
 from django.db import IntegrityError
 from django.contrib.auth.models import User
-from note.models import Note, NoteSpecial, NoteUser, NoteClub
+from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction
+from note.models import Note
 from note.models import Alias
-from note.models import TemplateCategory, TransactionTemplate,\
-    Transaction, RecurrentTransaction, MembershipTransaction, SpecialTransaction
-from member.models import Profile, Club, Membership
+from note.models import TemplateCategory, TransactionTemplate, \
+    Transaction, RecurrentTransaction, SpecialTransaction
+from member.models import Club
 
 """
 Script d'import de la nk15:
@@ -31,35 +30,40 @@ TODO: import ...
 
 """
 M_DURATION = 396
-M_START = datetime.date(2019,8,31)
-M_END = datetime.date(2020,9,30)
+M_START = datetime.date(2019, 8, 31)
+M_END = datetime.date(2020, 9, 30)
 
-
-MAP_IDBDE={
-    -4: 2, # Carte Bancaire
-    -3: 4, # Virement
-    -2: 1, # Especes
-    -1: 3, # Chèque
-     0: 5, # BDE
+MAP_IDBDE = {
+    -4: 2,  # Carte Bancaire
+    -3: 4,  # Virement
+    -2: 1,  # Especes
+    -1: 3,  # Chèque
+    0: 5,  # BDE
 }
 
-def update_line(n,N, content):
+MAP_IDACTIVITY = {}
+MAP_NAMEACTIVITY = {}
+MAP_NAMEGUEST = {}
+
+
+def update_line(n, total, content):
     n = str(n)
-    N = str(N)
-    n.rjust(len(N))
-    print(f"\r ({n}/{N}) {content:10.10}",end="")
+    total = str(total)
+    n.rjust(len(total))
+    print(f"\r ({n}/{total}) {content:10.10}", end="")
+
 
 @transaction.atomic
 def import_comptes(cur):
     cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
     pkclub = 3
-    N = cur.rowcount
+    n = cur.rowcount
     for idx, row in enumerate(cur):
-        update_line(idx,N,row["pseudo"])
+        update_line(idx, n, row["pseudo"])
         if row["type"] == "personne":
-            #sanitize password
+            # sanitize password
             if row["passwd"] != "*|*":
-                passwd_nk15 = "$".join(["custom_nk15","1",row["passwd"]])
+                passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
             else:
                 passwd_nk15 = ''
             try:
@@ -68,8 +72,8 @@ def import_comptes(cur):
                     "password": passwd_nk15,
                     "first_name": row["nom"],
                     "last_name": row["prenom"],
-                    "email":  row["mail"],
-                    "is_active" : True, # temporary
+                    "email": row["mail"],
+                    "is_active": True,  # temporary
                 }
 
                 user = User.objects.create(**obj_dict)
@@ -84,45 +88,45 @@ def import_comptes(cur):
                 # sanitize duplicate aliases (nk12)
             except ValidationError as e:
                 if e.code == 'same_alias':
-                    user.username = row["pseudo"]+str(row["idbde"])
+                    user.username = row["pseudo"] + str(row["idbde"])
                     user.save()
                 else:
-                    raise(e)
+                    raise e
             # profile  and note created via signal.
-            
+
             note = user.note
-            date = row.get("last_negatif",None)
-            if date != None:
+            date = row.get("last_negatif", None)
+            if date is not None:
                 note.last_negative = make_aware(date)
             note.balance = row["solde"]
-            obj_list =[user, profile, note]
             note.save()
-        else: # club
+        else:  # club
             obj_dict = {
-                "pk":pkclub,
+                "pk": pkclub,
                 "name": row["pseudo"],
                 "email": row["mail"],
                 "membership_duration": M_DURATION,
                 "membership_start": M_START,
                 "membership_end": M_END,
                 "membership_fee_paid": 0,
-                "membership_fee_unpaid":0,
+                "membership_fee_unpaid": 0,
             }
-            club,c = Club.objects.get_or_create(**obj_dict)
-            pkclub +=1
+            club, c = Club.objects.get_or_create(**obj_dict)
+            pkclub += 1
             note = club.note
             note.balance = row["solde"]
             club.save()
             note.save()
-            
+
         MAP_IDBDE[row["idbde"]] = note.note_ptr_id
 
+
 @transaction.atomic
 def import_boutons(cur):
     cur.execute("SELECT * FROM boutons;")
-    N = cur.rowcount
+    n = cur.rowcount
     for idx, row in enumerate(cur):
-        update_line(idx,N,row["label"])
+        update_line(idx, n, row["label"])
         cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"])
         if created:
             cat.save()
@@ -132,60 +136,60 @@ def import_boutons(cur):
             "amount": row["montant"],
             "destination_id": MAP_IDBDE[row["destinataire"]],
             "category": cat,
-            "display" : row["affiche"],
+            "display": row["affiche"],
             "description": row["description"],
         }
         try:
-            with transaction.atomic(): # required for error management
+            with transaction.atomic():  # required for error management
                 button = TransactionTemplate.objects.create(**obj_dict)
         except IntegrityError as e:
             # button with the same name is not possible in NK20.
             if "unique" in e.args[0]:
                 qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name')
                 note_name = qs[0]["name"]
-                #rename button name
-                obj_dict["name"] ="{} {}".format(obj_dict["name"],note_name)
+                # rename button name
+                obj_dict["name"] = "{} {}".format(obj_dict["name"], note_name)
                 button = TransactionTemplate.objects.create(**obj_dict)
             else:
-                raise(e)
+                raise e
         button.save()
 
+
 @transaction.atomic
 def import_transaction(cur):
-    idmin=58770
+    idmin = 58770
     cur.execute("SELECT *, transactions.date AS transac_date\
                    FROM transactions\
                    LEFT JOIN adhesions ON transactions.id = adhesions.id\
                    WHERE transactions.id> {}\
                    ORDER BY transactions.id;".format(idmin))
-    N = cur.rowcount
-    transac_list = []
+    n = cur.rowcount
     for idx, row in enumerate(cur):
-        update_line(idx,N,row["description"])
+        update_line(idx, n, row["description"])
         # some date are set to None, use the previous one
         date = row["transac_date"]
         obj_dict = {
-           # "pk": row["id"],
-            "destination_id" : MAP_IDBDE[row["destinataire"]],
+            # "pk": row["id"],
+            "destination_id": MAP_IDBDE[row["destinataire"]],
             "source_id": MAP_IDBDE[row["emetteur"]],
-            "created_at":make_aware(date),
-            "amount":row["montant"],
-            "quantity":row["quantite"],
-            "reason":row["description"],
-            "valid":row["valide"],
+            "created_at": make_aware(date),
+            "amount": row["montant"],
+            "quantity": row["quantite"],
+            "reason": row["description"],
+            "valid": row["valide"],
         }
         ttype = row["type"]
-        if ttype == "don" or ttype == "transfert" or ttype == "invitation":
-            transac = Transaction.objects.create(**obj_dict)
+        if ttype == "don" or ttype == "transfert":
+            Transaction.objects.create(**obj_dict)
         elif ttype == "bouton":
             cat_name = row["categorie"]
-            if cat_name == None:
+            if cat_name is None:
                 cat_name = 'None'
             cat, created = TemplateCategory.objects.get_or_create(name=cat_name)
             if created:
                 cat.save()
             obj_dict["category"] = cat
-            transac = RecurrentTransaction.objects.create(**obj_dict)
+            RecurrentTransaction.objects.create(**obj_dict)
         elif ttype == "crédit" or ttype == "retrait":
             field_id = "source_id" if ttype == "crédit" else "destination_id"
             if "espèce" in row["description"]:
@@ -206,84 +210,204 @@ def import_transaction(cur):
                 obj_dict["first_name"] = actor.club.name
                 obj_dict["last_name"] = actor.club.name
             else:
-                raise("You should'nt be there")
-            transac = SpecialTransaction.objects.create(**obj_dict)
+                raise Exception("You should'nt be there")
+            SpecialTransaction.objects.create(**obj_dict)
         elif ttype == "adhésion":
             print("adhesion not supported yet")
+        elif ttype == "invitation":
+            m = re.search("Invitation (.*?) \((.*?)\)", row["description"])
+            if m is None:
+                raise IntegrityError("Invitation is not well formated: {} (must be 'Invitation ACTIVITY_NAME (NAME)')"
+                                     .format(row["description"]))
+
+            activity_name = m.group(1)
+            guest_name = m.group(2)
+
+            if activity_name not in MAP_NAMEACTIVITY:
+                raise IntegrityError("Activity {} is not found".format(activity_name,))
+            activity = MAP_NAMEACTIVITY[activity_name]
+
+            if guest_name not in MAP_NAMEGUEST:
+                raise IntegrityError("Guest {} is not found".format(guest_name,))
+
+            guest = None
+            for g in MAP_NAMEGUEST[guest_name]:
+                if g.activity.pk == activity.pk:
+                    guest = g
+                    break
+            if guest is None:
+                raise IntegrityError("Guest {} didn't go to the activity {}".format(guest_name, activity_name,))
+
+            obj_dict["guest"] = guest
+
+            GuestTransaction.objects.get_or_create(**obj_dict)
         else:
-           print("other type not supported yet")
+            print("other type not supported yet:", ttype)
+
 
 @transaction.atomic
 def import_aliases(cur):
     cur.execute("SELECT * FROM aliases ORDER by id")
-    N = cur.rowcount
+    n = cur.rowcount
     for idx, row in enumerate(cur):
-        update_line(idx,N,row["alias"])
+        update_line(idx, n, row["titre"])
         alias_name = row["alias"]
-        alias_name_good = (alias_name[:252]+'...') if len(alias_name) > 255 else alias_name
+        alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
         obj_dict = {
-            "note_id":MAP_IDBDE[row["idbde"]],
-            "name":alias_name_good,
-            "normalized_name":Alias.normalize(alias_name_good)
+            "note_id": MAP_IDBDE[row["idbde"]],
+            "name": alias_name_good,
+            "normalized_name": Alias.normalize(alias_name_good)
         }
         try:
             with transaction.atomic():
-                alias, created =  Alias.objects.get_or_create(**obj_dict)
+                alias, created = Alias.objects.get_or_create(**obj_dict)
 
         except IntegrityError as e:
             if "unique" in e.args[0]:
                 continue
             else:
-                raise(e)
+                raise e
         alias.save()
 
 
+@transaction.atomic
+def import_activities(cur):
+    cur.execute("SELECT * FROM activites ORDER by id")
+    n = cur.rowcount
+    activity_type = ActivityType.objects.get(name="Pot")  # Need to be fixed manually
+    kfet = Club.objects.get(name="Kfet")
+    for idx, row in enumerate(cur):
+        update_line(idx, n, row["alias"])
+        organizer = Club.objects.filter(name=row["signature"])
+        if organizer.exists():
+            # Try to find the club that organizes the activity. If not founded, assume that is Kfet (fix manually)
+            organizer = organizer.get()
+        else:
+            organizer = kfet
+        obj_dict = {
+            "name": row["titre"],
+            "description": row["description"],
+            "activity_type": activity_type,  # By default Pot
+            "creater": MAP_IDBDE[row["responsable"]],
+            "organizer": organizer,
+            "attendees_club": kfet,  # Maybe fix manually
+            "date_start": row["debut"],
+            "date_end": row["fin"],
+            "valid": row["validepar"] is not None,
+            "open": row["open"],  # Should be always False
+        }
+        # WARNING: Fields lieu, liste, listeimprimee are missing
+        try:
+            with transaction.atomic():
+                activity = Activity.objects.get_or_create(**obj_dict)[0]
+                MAP_IDACTIVITY[row["id"]] = activity
+                MAP_NAMEACTIVITY[activity.name] = activity
+        except IntegrityError as e:
+            raise e
+
+
+@transaction.atomic
+def import_activity_entries(cur):
+    map_idguests = {}
+
+    cur.execute("SELECT * FROM invites ORDER by id")
+    n = cur.rowcount
+    for idx, row in enumerate(cur):
+        update_line(idx, n, row["nom"] + " " + row["prenom"])
+        obj_dict = {
+            "activity": MAP_IDACTIVITY[row["activity"]],
+            "last_name": row["nom"],
+            "first_name": row["prenom"],
+            "inviter": MAP_IDBDE[row["responsable"]],
+        }
+        try:
+            with transaction.atomic():
+                guest = Guest.objects.get_or_create(**obj_dict)[0]
+                map_idguests.setdefault(row["responsable"], [])
+                map_idguests[row["id"]].append(guest)
+                guest_name = guest.first_name + " " + guest.last_name
+                MAP_NAMEGUEST.setdefault(guest_name, [])
+                MAP_NAMEGUEST[guest_name].append(guest)
+        except IntegrityError as e:
+            raise e
+
+    cur.execute("SELECT * FROM entree_activites ORDER by id")
+    n = cur.rowcount
+    for idx, row in enumerate(cur):
+        update_line(idx, n, row["nom"] + " " + row["prenom"])
+        activity = MAP_IDACTIVITY[row["activity"]]
+        guest = None
+        if row["est_invite"]:
+            for g in map_idguests[row["id"]]:
+                if g.activity.pk == activity.pk:
+                    guest = g
+                    break
+            if not guest:
+                raise IntegrityError("Guest was not found: " + str(row))
+        obj_dict = {
+            "activity": activity,
+            "time": row["heure_entree"],
+            "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]],
+            "guest": guest,
+        }
+        try:
+            with transaction.atomic():
+                Entry.objects.get_or_create(**obj_dict)
+        except IntegrityError as e:
+            raise e
+
+
 class Command(BaseCommand):
     """
     Command for importing the database of NK15.
     Need to be run by a user with a registered role in postgres for the database nk15. 
     """
-    def print_success(self,to_print):
+
+    def print_success(self, to_print):
         return self.stdout.write(self.style.SUCCESS(to_print))
-    
-    def add_arguments(self,parser):
-        parser.add_argument('-c', '--comptes', action = 'store_true', help="import accounts")
-        parser.add_argument('-b', '--boutons', action = 'store_true', help="import boutons")
-        parser.add_argument('-t', '--transactions', action = 'store_true',help="import transaction")
-        parser.add_argument('-a', '--aliases', action = 'store_true',help="import aliases")
+
+    def add_arguments(self, parser):
+        parser.add_argument('-c', '--comptes', action='store_true', help="import accounts")
+        parser.add_argument('-b', '--boutons', action='store_true', help="import boutons")
+        parser.add_argument('-t', '--transactions', action='store_true', help="import transaction")
+        parser.add_argument('-al', '--aliases', action='store_true', help="import aliases")
+        parser.add_argument('-ac', '--activities', action='store_true', help="import activities")
         parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
         parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
         parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name')
         parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner')
-       
+
     def handle(self, *args, **kwargs):
         global MAP_IDBDE
         nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
         # connecting to nk15 database
-        conn = pg.connect(database=nk15db,user=nk15user)
-        cur = conn.cursor(cursor_factory = pge.DictCursor)
+        conn = pg.connect(database=nk15db, user=nk15user)
+        cur = conn.cursor(cursor_factory=pge.DictCursor)
 
         if kwargs["comptes"]:
-            #reset database.
+            # reset database.
             call_command("migrate")
-            call_command("loaddata","initial")
+            call_command("loaddata", "initial")
             self.print_success("reset  nk20 database\n")
             import_comptes(cur)
             self.print_success("comptes table imported")
         elif kwargs["map"]:
             filename = kwargs["map"]
-            with open(filename,'r') as fp:
+            with open(filename, 'r') as fp:
                 MAP_IDBDE = json.load(fp)
-                MAP_IDBDE = {int(k):int(v) for k,v in MAP_IDBDE.items()}
+                MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()}
         if kwargs["save"]:
             filename = kwargs["save"]
-            with open(filename,'w') as fp:
-                json.dump(MAP_IDBDE,fp,sort_keys=True, indent=2)
-                
+            with open(filename, 'w') as fp:
+                json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
+
         # /!\ need a prober MAP_IDBDE
         if kwargs["boutons"]:
             import_boutons(cur)
             self.print_success("boutons table imported\n")
+        if kwargs["activities"]:
+            import_activities(cur)
+            self.print_success("activities imported\n")
         if kwargs["aliases"]:
             import_aliases(cur)
             self.print_success("aliases imported\n")