From 4ce9aa0d4aef26ff0c1f8dbed6dffd3d37adab79 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 27 Apr 2020 01:19:56 +0200 Subject: [PATCH] Import activities, guests and guest transactions --- management/commands/import_nk15.py | 302 ++++++++++++++++++++--------- 1 file changed, 213 insertions(+), 89 deletions(-) diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index 5e70c47..31d7246 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -2,25 +2,24 @@ from django.core.management.base import BaseCommand from django.core.management import call_command -from django.utils import timezone -import psycopg2 as pg +import psycopg2 as pg import psycopg2.extras as pge from django.db import transaction import json import datetime -import collections +import re from django.core.exceptions import ValidationError from django.utils.timezone import make_aware - from django.db import IntegrityError from django.contrib.auth.models import User -from note.models import Note, NoteSpecial, NoteUser, NoteClub +from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction +from note.models import Note from note.models import Alias -from note.models import TemplateCategory, TransactionTemplate,\ - Transaction, RecurrentTransaction, MembershipTransaction, SpecialTransaction -from member.models import Profile, Club, Membership +from note.models import TemplateCategory, TransactionTemplate, \ + Transaction, RecurrentTransaction, SpecialTransaction +from member.models import Club """ Script d'import de la nk15: @@ -31,35 +30,40 @@ TODO: import ... """ M_DURATION = 396 -M_START = datetime.date(2019,8,31) -M_END = datetime.date(2020,9,30) +M_START = datetime.date(2019, 8, 31) +M_END = datetime.date(2020, 9, 30) - -MAP_IDBDE={ - -4: 2, # Carte Bancaire - -3: 4, # Virement - -2: 1, # Especes - -1: 3, # Chèque - 0: 5, # BDE +MAP_IDBDE = { + -4: 2, # Carte Bancaire + -3: 4, # Virement + -2: 1, # Especes + -1: 3, # Chèque + 0: 5, # BDE } -def update_line(n,N, content): +MAP_IDACTIVITY = {} +MAP_NAMEACTIVITY = {} +MAP_NAMEGUEST = {} + + +def update_line(n, total, content): n = str(n) - N = str(N) - n.rjust(len(N)) - print(f"\r ({n}/{N}) {content:10.10}",end="") + total = str(total) + n.rjust(len(total)) + print(f"\r ({n}/{total}) {content:10.10}", end="") + @transaction.atomic def import_comptes(cur): cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;") pkclub = 3 - N = cur.rowcount + n = cur.rowcount for idx, row in enumerate(cur): - update_line(idx,N,row["pseudo"]) + update_line(idx, n, row["pseudo"]) if row["type"] == "personne": - #sanitize password + # sanitize password if row["passwd"] != "*|*": - passwd_nk15 = "$".join(["custom_nk15","1",row["passwd"]]) + passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]]) else: passwd_nk15 = '' try: @@ -68,8 +72,8 @@ def import_comptes(cur): "password": passwd_nk15, "first_name": row["nom"], "last_name": row["prenom"], - "email": row["mail"], - "is_active" : True, # temporary + "email": row["mail"], + "is_active": True, # temporary } user = User.objects.create(**obj_dict) @@ -84,45 +88,45 @@ def import_comptes(cur): # sanitize duplicate aliases (nk12) except ValidationError as e: if e.code == 'same_alias': - user.username = row["pseudo"]+str(row["idbde"]) + user.username = row["pseudo"] + str(row["idbde"]) user.save() else: - raise(e) + raise e # profile and note created via signal. - + note = user.note - date = row.get("last_negatif",None) - if date != None: + date = row.get("last_negatif", None) + if date is not None: note.last_negative = make_aware(date) note.balance = row["solde"] - obj_list =[user, profile, note] note.save() - else: # club + else: # club obj_dict = { - "pk":pkclub, + "pk": pkclub, "name": row["pseudo"], "email": row["mail"], "membership_duration": M_DURATION, "membership_start": M_START, "membership_end": M_END, "membership_fee_paid": 0, - "membership_fee_unpaid":0, + "membership_fee_unpaid": 0, } - club,c = Club.objects.get_or_create(**obj_dict) - pkclub +=1 + club, c = Club.objects.get_or_create(**obj_dict) + pkclub += 1 note = club.note note.balance = row["solde"] club.save() note.save() - + MAP_IDBDE[row["idbde"]] = note.note_ptr_id + @transaction.atomic def import_boutons(cur): cur.execute("SELECT * FROM boutons;") - N = cur.rowcount + n = cur.rowcount for idx, row in enumerate(cur): - update_line(idx,N,row["label"]) + update_line(idx, n, row["label"]) cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"]) if created: cat.save() @@ -132,60 +136,60 @@ def import_boutons(cur): "amount": row["montant"], "destination_id": MAP_IDBDE[row["destinataire"]], "category": cat, - "display" : row["affiche"], + "display": row["affiche"], "description": row["description"], } try: - with transaction.atomic(): # required for error management + with transaction.atomic(): # required for error management button = TransactionTemplate.objects.create(**obj_dict) except IntegrityError as e: # button with the same name is not possible in NK20. if "unique" in e.args[0]: qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name') note_name = qs[0]["name"] - #rename button name - obj_dict["name"] ="{} {}".format(obj_dict["name"],note_name) + # rename button name + obj_dict["name"] = "{} {}".format(obj_dict["name"], note_name) button = TransactionTemplate.objects.create(**obj_dict) else: - raise(e) + raise e button.save() + @transaction.atomic def import_transaction(cur): - idmin=58770 + idmin = 58770 cur.execute("SELECT *, transactions.date AS transac_date\ FROM transactions\ LEFT JOIN adhesions ON transactions.id = adhesions.id\ WHERE transactions.id> {}\ ORDER BY transactions.id;".format(idmin)) - N = cur.rowcount - transac_list = [] + n = cur.rowcount for idx, row in enumerate(cur): - update_line(idx,N,row["description"]) + update_line(idx, n, row["description"]) # some date are set to None, use the previous one date = row["transac_date"] obj_dict = { - # "pk": row["id"], - "destination_id" : MAP_IDBDE[row["destinataire"]], + # "pk": row["id"], + "destination_id": MAP_IDBDE[row["destinataire"]], "source_id": MAP_IDBDE[row["emetteur"]], - "created_at":make_aware(date), - "amount":row["montant"], - "quantity":row["quantite"], - "reason":row["description"], - "valid":row["valide"], + "created_at": make_aware(date), + "amount": row["montant"], + "quantity": row["quantite"], + "reason": row["description"], + "valid": row["valide"], } ttype = row["type"] - if ttype == "don" or ttype == "transfert" or ttype == "invitation": - transac = Transaction.objects.create(**obj_dict) + if ttype == "don" or ttype == "transfert": + Transaction.objects.create(**obj_dict) elif ttype == "bouton": cat_name = row["categorie"] - if cat_name == None: + if cat_name is None: cat_name = 'None' cat, created = TemplateCategory.objects.get_or_create(name=cat_name) if created: cat.save() obj_dict["category"] = cat - transac = RecurrentTransaction.objects.create(**obj_dict) + RecurrentTransaction.objects.create(**obj_dict) elif ttype == "crédit" or ttype == "retrait": field_id = "source_id" if ttype == "crédit" else "destination_id" if "espèce" in row["description"]: @@ -206,84 +210,204 @@ def import_transaction(cur): obj_dict["first_name"] = actor.club.name obj_dict["last_name"] = actor.club.name else: - raise("You should'nt be there") - transac = SpecialTransaction.objects.create(**obj_dict) + raise Exception("You should'nt be there") + SpecialTransaction.objects.create(**obj_dict) elif ttype == "adhésion": print("adhesion not supported yet") + elif ttype == "invitation": + m = re.search("Invitation (.*?) \((.*?)\)", row["description"]) + if m is None: + raise IntegrityError("Invitation is not well formated: {} (must be 'Invitation ACTIVITY_NAME (NAME)')" + .format(row["description"])) + + activity_name = m.group(1) + guest_name = m.group(2) + + if activity_name not in MAP_NAMEACTIVITY: + raise IntegrityError("Activity {} is not found".format(activity_name,)) + activity = MAP_NAMEACTIVITY[activity_name] + + if guest_name not in MAP_NAMEGUEST: + raise IntegrityError("Guest {} is not found".format(guest_name,)) + + guest = None + for g in MAP_NAMEGUEST[guest_name]: + if g.activity.pk == activity.pk: + guest = g + break + if guest is None: + raise IntegrityError("Guest {} didn't go to the activity {}".format(guest_name, activity_name,)) + + obj_dict["guest"] = guest + + GuestTransaction.objects.get_or_create(**obj_dict) else: - print("other type not supported yet") + print("other type not supported yet:", ttype) + @transaction.atomic def import_aliases(cur): cur.execute("SELECT * FROM aliases ORDER by id") - N = cur.rowcount + n = cur.rowcount for idx, row in enumerate(cur): - update_line(idx,N,row["alias"]) + update_line(idx, n, row["titre"]) alias_name = row["alias"] - alias_name_good = (alias_name[:252]+'...') if len(alias_name) > 255 else alias_name + alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name obj_dict = { - "note_id":MAP_IDBDE[row["idbde"]], - "name":alias_name_good, - "normalized_name":Alias.normalize(alias_name_good) + "note_id": MAP_IDBDE[row["idbde"]], + "name": alias_name_good, + "normalized_name": Alias.normalize(alias_name_good) } try: with transaction.atomic(): - alias, created = Alias.objects.get_or_create(**obj_dict) + alias, created = Alias.objects.get_or_create(**obj_dict) except IntegrityError as e: if "unique" in e.args[0]: continue else: - raise(e) + raise e alias.save() +@transaction.atomic +def import_activities(cur): + cur.execute("SELECT * FROM activites ORDER by id") + n = cur.rowcount + activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually + kfet = Club.objects.get(name="Kfet") + for idx, row in enumerate(cur): + update_line(idx, n, row["alias"]) + organizer = Club.objects.filter(name=row["signature"]) + if organizer.exists(): + # Try to find the club that organizes the activity. If not founded, assume that is Kfet (fix manually) + organizer = organizer.get() + else: + organizer = kfet + obj_dict = { + "name": row["titre"], + "description": row["description"], + "activity_type": activity_type, # By default Pot + "creater": MAP_IDBDE[row["responsable"]], + "organizer": organizer, + "attendees_club": kfet, # Maybe fix manually + "date_start": row["debut"], + "date_end": row["fin"], + "valid": row["validepar"] is not None, + "open": row["open"], # Should be always False + } + # WARNING: Fields lieu, liste, listeimprimee are missing + try: + with transaction.atomic(): + activity = Activity.objects.get_or_create(**obj_dict)[0] + MAP_IDACTIVITY[row["id"]] = activity + MAP_NAMEACTIVITY[activity.name] = activity + except IntegrityError as e: + raise e + + +@transaction.atomic +def import_activity_entries(cur): + map_idguests = {} + + cur.execute("SELECT * FROM invites ORDER by id") + n = cur.rowcount + for idx, row in enumerate(cur): + update_line(idx, n, row["nom"] + " " + row["prenom"]) + obj_dict = { + "activity": MAP_IDACTIVITY[row["activity"]], + "last_name": row["nom"], + "first_name": row["prenom"], + "inviter": MAP_IDBDE[row["responsable"]], + } + try: + with transaction.atomic(): + guest = Guest.objects.get_or_create(**obj_dict)[0] + map_idguests.setdefault(row["responsable"], []) + map_idguests[row["id"]].append(guest) + guest_name = guest.first_name + " " + guest.last_name + MAP_NAMEGUEST.setdefault(guest_name, []) + MAP_NAMEGUEST[guest_name].append(guest) + except IntegrityError as e: + raise e + + cur.execute("SELECT * FROM entree_activites ORDER by id") + n = cur.rowcount + for idx, row in enumerate(cur): + update_line(idx, n, row["nom"] + " " + row["prenom"]) + activity = MAP_IDACTIVITY[row["activity"]] + guest = None + if row["est_invite"]: + for g in map_idguests[row["id"]]: + if g.activity.pk == activity.pk: + guest = g + break + if not guest: + raise IntegrityError("Guest was not found: " + str(row)) + obj_dict = { + "activity": activity, + "time": row["heure_entree"], + "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]], + "guest": guest, + } + try: + with transaction.atomic(): + Entry.objects.get_or_create(**obj_dict) + except IntegrityError as e: + raise e + + class Command(BaseCommand): """ Command for importing the database of NK15. Need to be run by a user with a registered role in postgres for the database nk15. """ - def print_success(self,to_print): + + def print_success(self, to_print): return self.stdout.write(self.style.SUCCESS(to_print)) - - def add_arguments(self,parser): - parser.add_argument('-c', '--comptes', action = 'store_true', help="import accounts") - parser.add_argument('-b', '--boutons', action = 'store_true', help="import boutons") - parser.add_argument('-t', '--transactions', action = 'store_true',help="import transaction") - parser.add_argument('-a', '--aliases', action = 'store_true',help="import aliases") + + def add_arguments(self, parser): + parser.add_argument('-c', '--comptes', action='store_true', help="import accounts") + parser.add_argument('-b', '--boutons', action='store_true', help="import boutons") + parser.add_argument('-t', '--transactions', action='store_true', help="import transaction") + parser.add_argument('-al', '--aliases', action='store_true', help="import aliases") + parser.add_argument('-ac', '--activities', action='store_true', help="import activities") parser.add_argument('-s', '--save', action='store', help="save mapping of idbde") parser.add_argument('-m', '--map', action='store', help="import mapping of idbde") parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name') parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner') - + def handle(self, *args, **kwargs): global MAP_IDBDE nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database - conn = pg.connect(database=nk15db,user=nk15user) - cur = conn.cursor(cursor_factory = pge.DictCursor) + conn = pg.connect(database=nk15db, user=nk15user) + cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["comptes"]: - #reset database. + # reset database. call_command("migrate") - call_command("loaddata","initial") + call_command("loaddata", "initial") self.print_success("reset nk20 database\n") import_comptes(cur) self.print_success("comptes table imported") elif kwargs["map"]: filename = kwargs["map"] - with open(filename,'r') as fp: + with open(filename, 'r') as fp: MAP_IDBDE = json.load(fp) - MAP_IDBDE = {int(k):int(v) for k,v in MAP_IDBDE.items()} + MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()} if kwargs["save"]: filename = kwargs["save"] - with open(filename,'w') as fp: - json.dump(MAP_IDBDE,fp,sort_keys=True, indent=2) - + with open(filename, 'w') as fp: + json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2) + # /!\ need a prober MAP_IDBDE if kwargs["boutons"]: import_boutons(cur) self.print_success("boutons table imported\n") + if kwargs["activities"]: + import_activities(cur) + self.print_success("activities imported\n") if kwargs["aliases"]: import_aliases(cur) self.print_success("aliases imported\n")