From 11e6bd972093abc5792a55526b9fb5151fdfd1ac Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Thu, 21 May 2020 18:28:00 +0200 Subject: [PATCH] add basic support for transaction and activity, not tested! --- management/commands/import_activities.py | 111 ++++++++++++ management/commands/import_transaction.py | 204 ++++++++++++++++++++++ 2 files changed, 315 insertions(+) create mode 100644 management/commands/import_activities.py create mode 100644 management/commands/import_transaction.py diff --git a/management/commands/import_activities.py b/management/commands/import_activities.py new file mode 100644 index 0000000..04f5268 --- /dev/null +++ b/management/commands/import_activities.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +import psycopg2 as pg +import psycopg2.extras as pge +import pytz +import datetime +import copy + +from django.utils.timezone import make_aware +from django.db import transaction + +from activity.models import ActivityType, Activity +from member.models import Club + +from ._import_utils import ImportCommand, BulkCreateManager + + +class Command(ImportCommand): + """ + Import command for Activities Base Data (Comptes, and Aliases) + """ + + def add_arguments(self, parser): + pass + + @transaction.atomic + def import_activities(self, cur, chunk_size): + cur.execute("SELECT * FROM activites ORDER by id") + n = cur.rowcount + bulk_mgr = BulkCreateManager(chunk_size=chunk_size) + activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually + kfet = Club.objects.get(name="Kfet") + for idx, row in enumerate(cur): + update_line(idx, n, row["alias"]) + organizer = Club.objects.filter(name=row["signature"]) + if organizer.exists(): + # Try to find the club that organizes the activity. If not found, assume it's Kfet (fix manually) + organizer = organizer.get() + else: + organizer = kfet + obj_dict = { + "pk": row["id"] + "name": row["titre"], + "description": row["description"], + "activity_type": activity_type, # By default Pot + "creater": self.MAP_IDBDE[row["responsable"]], + "organizer": organizer, + "attendees_club": kfet, # Maybe fix manually + "date_start": row["debut"], + "date_end": row["fin"], + "valid": row["validepar"] is not None, + "open": row["open"], # Should always be False + } + # WARNING: Fields lieu, liste, listeimprimee are missing + # + bulk_mgr.add(Activity(**obj_dict)) + MAP_NAMEACTIVITY[activity.name] = activity + bulk_mgr.done() + return MAP_IDACTIVITY, MAP_NAMEACTIVITY + + @transaction.atomic + def import_activity_entries(cur): + bulk_mgr = BulkCreateManager() + map_idguests = set() + cur.execute("SELECT * FROM invites ORDER by id") + n = cur.rowcount + for idx, row in enumerate(cur): + update_line(idx, n, row["nom"] + " " + row["prenom"]) + obj_dict = { + "pk": row["id"], + "activity_id": row["activity"], + "last_name": row["nom"], + "first_name": row["prenom"], + "inviter": self.MAP_IDBDE[row["responsable"]], + } + bulk_mgr.add(Guest(**obj_dict)) + bulk_mgr.done() + cur.execute("SELECT * FROM entree_activites ORDER by id") + n = cur.rowcount + for idx, row in enumerate(cur): + update_line(idx, n, row["nom"] + " " + row["prenom"]) + guest = None + if row["est_invite"]: + for g in map_idguests[row["id"]]: + if g.activity.pk == activity.pk: + guest = g + break + if not guest: + obj_dict = { + "activity": row["activity"], + "time": make_aware(row["heure_entree"]), + "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]], + "guest": guest, + } + try: + with transaction.atomic(): + Entry.objects.get_or_create(**obj_dict) + except IntegrityError as e: + raise e + + + def handle(self, *args, **kwargs): + # default args, provided by ImportCommand. + nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] + # connecting to nk15 database + conn = pg.connect(database=nk15db, user=nk15user) + cur = conn.cursor(cursor_factory=pge.DictCursor) + + if kwargs["map"]: + self.load(kwargs["map"]) + self.import_activities(cur, chunk_size) diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py new file mode 100644 index 0000000..03a21c2 --- /dev/null +++ b/management/commands/import_transaction.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 + +import psycopg2 as pg +import psycopg2.extras as pge +import pytz +import datetime +import copy + +from django.utils.timezone import make_aware +from django.db import transaction + +from note.models import (TemplateCategory, + TransactionTemplate, + Transaction, + RecurrentTransaction, + SpecialTransaction + ) +from note.models import Note + +from member.models import Membership, MembershipTransaction +from ._import_utils import ImportCommand, BulkCreateManager + +BDE_PK = 1 +KFET_PK = 2 +NOTE_SPECIAL_CODE = { + "espèce": 1, + "carte": 2, + "chèque": 3, + "virement": 4, +} + + +def get_date_end(date_start): + date_end = copy.deepcopy(date_start) + if date_start > 8: + date_end.year = date_start + 1 + date_end.month = 9 + date_end.day = 30 + return date_end + + +class Command(ImportCommand): + """ + Import command for People base data (Comptes, and Aliases) + """ + + def add_arguments(self, parser): + parser.add_argument('-b', '--buttons', action='store_true', help="import buttons") + parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import") + + @transaction.atomic + def import_buttons(self, cur, chunk_size): + categories = dict() + buttons = dict() + bulk_mgr = BulkCreateManager(chunk_size=chunk_size) + cur.execute("SELECT * FROM boutons;") + n = cur.rowcount + pk_category = 1 + for idx, row in enumerate(cur): + self.update_line(idx, n, row["label"]) + if row["categorie"] not in categories: + bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"])) + pk_category += 1 + categories[row["categorie"]] = pk_category + obj_dict = { + "pk": row["id"], + "name": row["label"], + "amount": row["montant"], + "destination_id": self.MAP_IDBDE[row["destinataire"]], + "category_id": categories[row["categorie"]], + "display": row["affiche"], + "description": row["description"], + } + if row["label"] in buttons: + obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}" + bulk_mgr.add(TransactionTemplate(**obj_dict)) + buttons[obj_dict["label"]] = row["id"] + bulk_mgr.done() + return buttons, categories + + @transaction.atomic + def import_transaction(self, cur, chunk_size, idmin, buttons, categories): + bulk_mgr = BulkCreateManager(chunk_size=chunk_size) + cur.execute( + f"SELECT t.date AS transac_date, t.type, t.emetteur,\ + t.destinataire,t.quantite, t.montant, t.description,\ + t.valide, t.cantinvalidate, t.categorie, \ + a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\ + FROM transactions AS t \ + LEFT JOIN adhesions AS a ON t.id = a.idtransaction \ + WHERE t.id >= {idmin} \ + ORDER BY t.id;") + n = cur.rowcount + pk_membership = 1 + pk_transaction = 1 + for idx, row in enumerate(cur): + self.update_line(idx, n, row["description"]) + try: + date = make_aware(row["transac_date"]) + except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError): + date = make_aware(row["transac_date"] + datetime.timedelta(hours=1)) + + # standart transaction object + obj_dict = { + "pk": pk_transaction, + "destination_id": self.MAP_IDBDE[row["destinataire"]], + "source_id": self.MAP_IDBDE[row["emetteur"]], + "created_at": date, + "amount": row["montant"], + "quantity": row["quantite"], + "reason": row["description"], + "valid": row["valide"], + } + # for child transaction Models + child_dict = {"pk": obj_dict["pk"]} + ttype = row["type"] + if ttype == "don" or ttype == "transfert": + child_transaction = None + elif ttype == "bouton": + child_transaction = RecurrentTransaction + child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"]) + child_dict["template_id"] = buttons[row["description"]] + elif ttype == "crédit" or ttype == "retrait": + child_transaction = SpecialTransaction + # Some transaction uses BDE (idbde=0) as source or destination, + # lets fix that. + field_id = "source_id" if ttype == "crédit" else "destination_id" + if "espèce" in row["description"]: + obj_dict[field_id] = 1 + elif "carte" in row["description"]: + obj_dict[field_id] = 2 + elif "cheques" in row["description"]: + obj_dict[field_id] = 3 + elif "virement" in row["description"]: + obj_dict[field_id] = 4 + # humans and clubs have always the biggest id + actor_pk = max(row["destinataire"], row["emetteur"]) + actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk]) + # custom fields of SpecialTransaction + if actor.__class__.__name__ == "NoteUser": + child_dict["first_name"] = actor.user.first_name + child_dict["last_name"] = actor.user.last_name + else: + child_dict["first_name"] = actor.club.name + child_dict["last_name"] = actor.club.name + elif ttype == "adhésion" and row["valide"]: + child_transaction = MembershipTransaction + # Kfet membership + montant = row["montant"] + obj_dict["amount"] = min(500, montant) + child_dict["membership_id"] = pk_membership + kfet_dict = { + "pk": pk_membership, + "user": self.MAP_IDBDE[row["idbde"]], + "club": KFET_PK, + "date_start": row["date"].date(), # Only date, not time + "date_end": get_date_end(row["date"].date()), + "fee": min(500, montant) + } + + pk_membership += 1 + pk_transaction += 1 + # BDE Membership + obj_dict2 = obj_dict.copy() + child_dict2 = dict() + obj_dict2["pk"] = pk_transaction + obj_dict2["amount"] = max(montant - 500, 0) + child_dict2["pk"] = pk_transaction + bde_dict = { + "pk": pk_membership, + "user": self.MAP_IDBDE[row["idbde"]], + "club": BDE_PK, + "date_start": row["date"].date(), # Only date, not time + "date_end": get_date_end(row["date"].date()), + "fee": max(montant - 500, 0), + } + pk_membership += 1 + # BDE membership Transaction is inserted before the Kfet membershipTransaction + bulk_mgr.add( + Transaction(**obj_dict2), + child_transaction(**child_dict2), + Membership(**bde_dict), + Membership(**kfet_dict), + ) + elif ttype == "invitation": + print("invitation not supported yet:", ttype) + + bulk_mgr.add(Transaction(**obj_dict), + child_transaction(**child_dict)) + pk_transaction += 1 + + def handle(self, *args, **kwargs): + # default args, provided by ImportCommand. + nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] + # connecting to nk15 database + conn = pg.connect(database=nk15db, user=nk15user) + cur = conn.cursor(cursor_factory=pge.DictCursor) + + if kwargs["map"]: + self.load(kwargs["map"]) + + self.import_buttons(cur, kwargs["chunk"]) + + self.import_transaction(cur, kwargs["chunk"])