#!/usr/env/bin python3 import json import datetime import re import pytz import psycopg2 as pg import psycopg2.extras as pge from django.core.management.base import BaseCommand from django.core.management import call_command from django.db import transaction from django.core.exceptions import ValidationError from django.utils.timezone import make_aware from django.db import IntegrityError from django.contrib.auth.models import User from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction from note.models import Note from note.models import Alias from note.models import ( TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, MembershipTransaction, SpecialTransaction, ) from member.models import Club, Membership from treasury.models import RemittanceType, Remittance, SpecialTransactionProxy """ Script d'import de la nk15: TODO: import transactions TODO: import adhesion TODO: import activite TODO: import ... """ M_DURATION = 396 M_START = datetime.date(2019, 8, 31) M_END = datetime.date(2020, 9, 30) MAP_IDBDE = { -4: 2, # Carte Bancaire -3: 4, # Virement -2: 1, # Especes -1: 3, # Chèque 0: 5, # BDE } MAP_IDACTIVITY = {} MAP_NAMEACTIVITY = {} MAP_NAMEGUEST = {} MAP_IDSPECIALTRANSACTION = {} def update_line(n, total, content): n = str(n) total = str(total) n.rjust(len(total)) print(f"\r ({n}/{total}) {content:10.10}", end="") @transaction.atomic def import_comptes(cur): cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;") pkclub = 3 n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["pseudo"]) if row["type"] == "personne": # sanitize password if row["passwd"] != "*|*" and not row["deleted"]: passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]]) else: passwd_nk15 = '' try: obj_dict = { "username": row["pseudo"], "password": passwd_nk15, "first_name": row["nom"], "last_name": row["prenom"], "email": row["mail"], "is_active": True, # temporary } user = User.objects.create(**obj_dict) profile = user.profile profile.phone_number = row['tel'] profile.address = row['adresse'] profile.paid = row['normalien'] profile.registration_valid = True profile.email_confirmed = True user.save() profile.save() # sanitize duplicate aliases (nk12) except ValidationError as e: if e.code == 'same_alias': user.username = row["pseudo"] + str(row["idbde"]) user.save() else: raise e # profile and note created via signal. note = user.note date = row.get("last_negatif", None) if date is not None: note.last_negative = make_aware(date) note.balance = row["solde"] note.save() else: # club obj_dict = { "pk": pkclub, "name": row["pseudo"], "email": row["mail"], "membership_duration": M_DURATION, "membership_start": M_START, "membership_end": M_END, "membership_fee_paid": 0, "membership_fee_unpaid": 0, } club, c = Club.objects.get_or_create(**obj_dict) pkclub += 1 note = club.note note.balance = row["solde"] club.save() note.save() MAP_IDBDE[row["idbde"]] = note.note_ptr_id @transaction.atomic def import_boutons(cur): cur.execute("SELECT * FROM boutons;") n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["label"]) cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"]) if created: cat.save() obj_dict = { "pk": row["id"], "name": row["label"], "amount": row["montant"], "destination_id": MAP_IDBDE[row["destinataire"]], "category": cat, "display": row["affiche"], "description": row["description"], } try: with transaction.atomic(): # required for error management button = TransactionTemplate.objects.create(**obj_dict) except IntegrityError as e: # button with the same name is not possible in NK20. if "unique" in e.args[0]: qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name') note_name = qs[0]["name"] # rename button name obj_dict["name"] = f"{obj_dict_name['name']} {note_name}" button = TransactionTemplate.objects.create(**obj_dict) else: raise e button.save() @transaction.atomic def import_transaction(cur): idmin = 58770 bde = Club.objects.get(name="BDE") kfet = Club.objects.get(name="Kfet") cur.execute( "SELECT t.date AS transac_date, t.type, t.emetteur,\ t.destinataire,t.quantite, t.montant, t.description,\ t.valide, t.cantinvalidate, t.categorie, \ a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ FROM transactions AS t \ LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ WHERE t.id> {} \ ORDER BY t.id;".format(idmin) ) n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["description"]) try: date = make_aware(row["transac_date"]) except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) # standart transaction object obj_dict = { # "pk": row["id"], "destination_id": MAP_IDBDE[row["destinataire"]], "source_id": MAP_IDBDE[row["emetteur"]], "created_at": make_aware(date), "amount": row["montant"], "quantity": row["quantite"], "reason": row["description"], "valid": row["valide"], } ttype = row["type"] if ttype == "don" or ttype == "transfert": Transaction.objects.create(**obj_dict) elif ttype == "bouton": cat_name = row["categorie"] if cat_name is None: cat_name = 'None' cat, created = TemplateCategory.objects.get_or_create(name=cat_name) if created: cat.save() obj_dict["category"] = cat RecurrentTransaction.objects.create(**obj_dict) elif ttype == "crédit" or ttype == "retrait": field_id = "source_id" if ttype == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 elif "carte" in row["description"]: obj_dict[field_id] = 2 elif "cheques" in row["description"]: obj_dict[field_id] = 3 elif "virement" in row["description"]: obj_dict[field_id] = 4 pk = max(row["destinataire"], row["emetteur"]) actor = Note.objects.get(id=MAP_IDBDE[pk]) # custom fields of SpecialTransaction if actor.__class__.__name__ == "NoteUser": obj_dict["first_name"] = actor.user.first_name obj_dict["last_name"] = actor.user.last_name elif actor.__class__.__name__ == "NoteClub": obj_dict["first_name"] = actor.club.name obj_dict["last_name"] = actor.club.name else: raise Exception("Badly formatted Special Transaction You should'nt be there.") tr = SpecialTransaction.objects.create(**obj_dict) if "cheques" in row["description"]: MAP_IDSPECIALTRANSACTION[row["id"]] = tr elif ttype == "adhésion": montant = row["montant"] # Create Double membership to Kfet and Bde # sometimes montant = 0, fees are modified accordingly. bde_dict = { "user": MAP_IDBDE[row["idbde"]], "club": bde, "date_start": row["date"].date(), # Only date, not time "fee": min(500, montant) } kfet_dict = { "user": MAP_IDBDE[row["idbde"]], "club": kfet, "date_start": row["date"].date(), # Only date, not time "fee": max(montant - 500, 0), } if row["valide"]: with transaction.atomic(): # membership save triggers MembershipTransaction creation bde_membership = Membership.objects.get_or_create(**bde_dict) kfet_membership = Membership.objects.get_or_create(**kfet_dict) bde_membership.transaction.created_at = row["transac_date"] bde_membership.transaction.description = row["description"] bde_membership.transaction.save() kfet_membership.transaction.created_at = row["transac_date"] kfet_membership.transaction.description = row["description"] + "(Kfet)" kfet_membership.transaction.save() else: # don't create membership MembershipTransaction.objects.create(**obj_dict) elif ttype == "invitation": m = re.search(r"Invitation (.*?) \((.*?)\)", row["description"]) if m is None: raise IntegrityError(f"Invitation is not well formated: {row['description']} (must be 'Invitation ACTIVITY_NAME (NAME)')") activity_name = m.group(1) guest_name = m.group(2) if activity_name not in MAP_NAMEACTIVITY: raise IntegrityError(f"Activity {activity_name} is not found") activity = MAP_NAMEACTIVITY[activity_name] if guest_name not in MAP_NAMEGUEST: raise IntegrityError(f"Guest {guest_name} is not found") guest = None for g in MAP_NAMEGUEST[guest_name]: if g.activity.pk == activity.pk: guest = g break if guest is None: raise IntegrityError("Guest {guest_name} didn't go to the activity {activity_name}") obj_dict["guest"] = guest GuestTransaction.objects.get_or_create(**obj_dict) else: print("other type not supported yet:", ttype) @transaction.atomic def import_aliases(cur): cur.execute("SELECT * FROM aliases ORDER by id") n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["alias"]) alias_name = row["alias"] alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name obj_dict = { "note_id": MAP_IDBDE[row["idbde"]], "name": alias_name_good, "normalized_name": Alias.normalize(alias_name_good), } try: with transaction.atomic(): alias, created = Alias.objects.get_or_create(**obj_dict) except IntegrityError as e: if "unique" in e.args[0]: continue else: raise e alias.save() @transaction.atomic def import_activities(cur): cur.execute("SELECT * FROM activites ORDER by id") n = cur.rowcount activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually kfet = Club.objects.get(name="Kfet") for idx, row in enumerate(cur): update_line(idx, n, row["alias"]) organizer = Club.objects.filter(name=row["signature"]) if organizer.exists(): # Try to find the club that organizes the activity. If not founded, assume it's Kfet (fix manually) organizer = organizer.get() else: organizer = kfet obj_dict = { "name": row["titre"], "description": row["description"], "activity_type": activity_type, # By default Pot "creater": MAP_IDBDE[row["responsable"]], "organizer": organizer, "attendees_club": kfet, # Maybe fix manually "date_start": row["debut"], "date_end": row["fin"], "valid": row["validepar"] is not None, "open": row["open"], # Should be always False } # WARNING: Fields lieu, liste, listeimprimee are missing try: with transaction.atomic(): activity = Activity.objects.get_or_create(**obj_dict)[0] MAP_IDACTIVITY[row["id"]] = activity MAP_NAMEACTIVITY[activity.name] = activity except IntegrityError as e: raise e @transaction.atomic def import_activity_entries(cur): map_idguests = {} cur.execute("SELECT * FROM invites ORDER by id") n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["nom"] + " " + row["prenom"]) obj_dict = { "activity": MAP_IDACTIVITY[row["activity"]], "last_name": row["nom"], "first_name": row["prenom"], "inviter": MAP_IDBDE[row["responsable"]], } try: with transaction.atomic(): guest = Guest.objects.get_or_create(**obj_dict)[0] map_idguests.setdefault(row["responsable"], []) map_idguests[row["id"]].append(guest) guest_name = guest.first_name + " " + guest.last_name MAP_NAMEGUEST.setdefault(guest_name, []) MAP_NAMEGUEST[guest_name].append(guest) except IntegrityError as e: raise e cur.execute("SELECT * FROM entree_activites ORDER by id") n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["nom"] + " " + row["prenom"]) activity = MAP_IDACTIVITY[row["activity"]] guest = None if row["est_invite"]: for g in map_idguests[row["id"]]: if g.activity.pk == activity.pk: guest = g break if not guest: raise IntegrityError("Guest was not found: " + str(row)) obj_dict = { "activity": activity, "time": row["heure_entree"], "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]], "guest": guest, } try: with transaction.atomic(): Entry.objects.get_or_create(**obj_dict) except IntegrityError as e: raise e @transaction.atomic def import_remittances(cur): cur.execute("SELECT * FROM remises ORDER by id") map_idremittance = {} n = cur.rowcount check_type = RemittanceType.objects.get(note__name="Chèque") for idx, row in enumerate(cur): update_line(idx, n, row["date"]) obj_dict = { "date": row["date"][10:], "remittance_type": check_type, "comment": row["commentaire"], "closed": row["close"], } try: with transaction.atomic(): remittance = Remittance.objects.get_or_create(**obj_dict) map_idremittance[row["id"]] = remittance except IntegrityError as e: raise e print("remittances are imported") print("imported checks") cur.execute("SELECT * FROM cheques ORDER by id") n = cur.rowcount for idx, row in enumerate(cur): update_line(idx, n, row["date"]) obj_dict = { "date": row["date"][10:], "remittance_type": check_type, "comment": row["commentaire"], "closed": row["close"], } tr = MAP_IDSPECIALTRANSACTION[row["idtransaction"]] proxy = SpecialTransactionProxy.objects.get_or_create(transaction=tr) proxy.remittance = map_idremittance[row["idremise"]] try: with transaction.atomic(): proxy.save() except IntegrityError as e: raise e class Command(BaseCommand): """ Command for importing the database of NK15. Need to be run by a user with a registered role in postgres for the database nk15. """ def print_success(self, to_print): return self.stdout.write(self.style.SUCCESS(to_print)) def add_arguments(self, parser): parser.add_argument('-c', '--comptes', action='store_true', help="import accounts") parser.add_argument('-b', '--boutons', action='store_true', help="import boutons") parser.add_argument('-t', '--transactions', action='store_true', help="import transaction") parser.add_argument('-al', '--aliases', action='store_true', help="import aliases") parser.add_argument('-ac', '--activities', action='store_true', help="import activities") parser.add_argument('-r', '--remittances', action='store_true', help="import check remittances") parser.add_argument('-s', '--save', action='store', help="save mapping of idbde") parser.add_argument('-m', '--map', action='store', help="import mapping of idbde") parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name') parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner') def handle(self, *args, **kwargs): global MAP_IDBDE nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) if kwargs["comptes"]: # reset database. call_command("migrate") call_command("loaddata", "initial") self.print_success("reset nk20 database\n") import_comptes(cur) self.print_success("comptes table imported") elif kwargs["map"]: filename = kwargs["map"] with open(filename, 'r') as fp: MAP_IDBDE = json.load(fp) MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()} if kwargs["save"]: filename = kwargs["save"] with open(filename, 'w') as fp: json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2) # /!\ need a prober MAP_IDBDE if kwargs["boutons"]: import_boutons(cur) self.print_success("boutons table imported\n") if kwargs["activities"]: import_activities(cur) self.print_success("activities imported\n") import_activity_entries(cur) self.print_success("activity entries imported\n") if kwargs["aliases"]: import_aliases(cur) self.print_success("aliases imported\n") if kwargs["transactions"]: import_transaction(cur) self.print_success("transaction imported\n") if kwargs["remittances"]: import_remittances(cur) self.print_success("remittances imported\n")