nk20-scripts/management/commands/import_nk15.py

#!/usr/env/bin python3

import json
import datetime
import re
import pytz

import psycopg2 as pg
import psycopg2.extras as pge

from django.core.management.base import BaseCommand
from django.core.management import call_command
from django.db import transaction
from django.core.exceptions import ValidationError
from django.utils.timezone import make_aware
from django.db import IntegrityError
from django.contrib.auth.models import User

from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction
from note.models import Note
from note.models import Alias
from note.models import (
    TemplateCategory,
    TransactionTemplate,
    Transaction,
    RecurrentTransaction,
    MembershipTransaction,
    SpecialTransaction,
)
from member.models import Club, Membership
from treasury.models import RemittanceType, Remittance, SpecialTransactionProxy

"""
Script d'import de la nk15:
TODO: import transactions
TODO: import adhesion
TODO: import activite
TODO: import ...

"""
M_DURATION = 396
M_START = datetime.date(2019, 8, 31)
M_END = datetime.date(2020, 9, 30)

MAP_IDBDE = {
    -4: 2,  # Carte Bancaire
    -3: 4,  # Virement
    -2: 1,  # Especes
    -1: 3,  # Chèque
    0: 5,  # BDE
}

MAP_IDACTIVITY = {}
MAP_NAMEACTIVITY = {}
MAP_NAMEGUEST = {}
MAP_IDSPECIALTRANSACTION = {}


def update_line(n, total, content):
    n = str(n)
    total = str(total)
    n.rjust(len(total))
    print(f"\r ({n}/{total}) {content:10.10}", end="")


@transaction.atomic
def import_comptes(cur):
    cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
    pkclub = 3
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["pseudo"])
        if row["type"] == "personne":
            # sanitize password
            if row["passwd"] != "*|*" and not row["deleted"]:
                passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
            else:
                passwd_nk15 = ''
            try:
                obj_dict = {
                    "username": row["pseudo"],
                    "password": passwd_nk15,
                    "first_name": row["nom"],
                    "last_name": row["prenom"],
                    "email": row["mail"],
                    "is_active": True,  # temporary
                }

                user = User.objects.create(**obj_dict)
                profile = user.profile
                profile.phone_number = row['tel']
                profile.address = row['adresse']
                profile.paid = row['normalien']
                profile.registration_valid = True
                profile.email_confirmed = True
                user.save()
                profile.save()
                # sanitize duplicate aliases (nk12)
            except ValidationError as e:
                if e.code == 'same_alias':
                    user.username = row["pseudo"] + str(row["idbde"])
                    user.save()
                else:
                    raise e
            # profile  and note created via signal.

            note = user.note
            date = row.get("last_negatif", None)
            if date is not None:
                note.last_negative = make_aware(date)
            note.balance = row["solde"]
            note.save()
        else:  # club
            obj_dict = {
                "pk": pkclub,
                "name": row["pseudo"],
                "email": row["mail"],
                "membership_duration": M_DURATION,
                "membership_start": M_START,
                "membership_end": M_END,
                "membership_fee_paid": 0,
                "membership_fee_unpaid": 0,
            }
            club, c = Club.objects.get_or_create(**obj_dict)
            pkclub += 1
            note = club.note
            note.balance = row["solde"]
            club.save()
            note.save()

        MAP_IDBDE[row["idbde"]] = note.note_ptr_id


@transaction.atomic
def import_boutons(cur):
    cur.execute("SELECT * FROM boutons;")
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["label"])
        cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"])
        if created:
            cat.save()
        obj_dict = {
            "pk": row["id"],
            "name": row["label"],
            "amount": row["montant"],
            "destination_id": MAP_IDBDE[row["destinataire"]],
            "category": cat,
            "display": row["affiche"],
            "description": row["description"],
        }
        try:
            with transaction.atomic():  # required for error management
                button = TransactionTemplate.objects.create(**obj_dict)
        except IntegrityError as e:
            # button with the same name is not possible in NK20.
            if "unique" in e.args[0]:
                qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name')
                note_name = qs[0]["name"]
                # rename button name
                obj_dict["name"] = f"{obj_dict_name['name']} {note_name}"
                button = TransactionTemplate.objects.create(**obj_dict)
            else:
                raise e
        button.save()


@transaction.atomic
def import_transaction(cur):
    idmin = 58770
    bde = Club.objects.get(name="BDE")
    kfet = Club.objects.get(name="Kfet")
    cur.execute(
        "SELECT t.date AS transac_date, t.type, t.emetteur,\
                t.destinataire,t.quantite, t.montant, t.description,\
                t.valide, t.cantinvalidate, t.categorie, \
                a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
         FROM transactions AS t \
           LEFT JOIN adhesions  AS a ON t.id = a.idtransaction \
         WHERE t.id> {} \
         ORDER BY t.id;".format(idmin)
    )
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["description"])
        try:
            date = make_aware(row["transac_date"])
        except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
            date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))

        # standart transaction object
        obj_dict = {
            # "pk": row["id"],
            "destination_id": MAP_IDBDE[row["destinataire"]],
            "source_id": MAP_IDBDE[row["emetteur"]],
            "created_at": make_aware(date),
            "amount": row["montant"],
            "quantity": row["quantite"],
            "reason": row["description"],
            "valid": row["valide"],
        }
        ttype = row["type"]
        if ttype == "don" or ttype == "transfert":
            Transaction.objects.create(**obj_dict)
        elif ttype == "bouton":
            cat_name = row["categorie"]
            if cat_name is None:
                cat_name = 'None'
            cat, created = TemplateCategory.objects.get_or_create(name=cat_name)
            if created:
                cat.save()
            obj_dict["category"] = cat
            RecurrentTransaction.objects.create(**obj_dict)
        elif ttype == "crédit" or ttype == "retrait":
            field_id = "source_id" if ttype == "crédit" else "destination_id"
            if "espèce" in row["description"]:
                obj_dict[field_id] = 1
            elif "carte" in row["description"]:
                obj_dict[field_id] = 2
            elif "cheques" in row["description"]:
                obj_dict[field_id] = 3
            elif "virement" in row["description"]:
                obj_dict[field_id] = 4
            pk = max(row["destinataire"], row["emetteur"])
            actor = Note.objects.get(id=MAP_IDBDE[pk])
            # custom fields of SpecialTransaction
            if actor.__class__.__name__ == "NoteUser":
                obj_dict["first_name"] = actor.user.first_name
                obj_dict["last_name"] = actor.user.last_name
            elif actor.__class__.__name__ == "NoteClub":
                obj_dict["first_name"] = actor.club.name
                obj_dict["last_name"] = actor.club.name
            else:
                raise Exception("Badly formatted Special Transaction You should'nt be there.")
            tr = SpecialTransaction.objects.create(**obj_dict)
            if "cheques" in row["description"]:
                MAP_IDSPECIALTRANSACTION[row["id"]] = tr

        elif ttype == "adhésion":
            montant = row["montant"]

            # Create Double membership to Kfet and Bde
            # sometimes montant = 0,  fees are modified accordingly.
            bde_dict = {
                "user": MAP_IDBDE[row["idbde"]],
                "club": bde,
                "date_start": row["date"].date(),  # Only date, not time
                "fee": min(500, montant)
            }
            kfet_dict = {
                "user": MAP_IDBDE[row["idbde"]],
                "club": kfet,
                "date_start": row["date"].date(),  # Only date, not time
                "fee": max(montant - 500, 0),
            }

            if row["valide"]:
                with transaction.atomic():
                    # membership save triggers MembershipTransaction creation
                    bde_membership = Membership.objects.get_or_create(**bde_dict)
                    kfet_membership = Membership.objects.get_or_create(**kfet_dict)
                    bde_membership.transaction.created_at = row["transac_date"]
                    bde_membership.transaction.description = row["description"]
                    bde_membership.transaction.save()
                    kfet_membership.transaction.created_at = row["transac_date"]
                    kfet_membership.transaction.description = row["description"] + "(Kfet)"
                    kfet_membership.transaction.save()
            else:
                # don't create membership
                MembershipTransaction.objects.create(**obj_dict)
        elif ttype == "invitation":
            m = re.search(r"Invitation (.*?) \((.*?)\)", row["description"])
            if m is None:
                raise IntegrityError(f"Invitation is not well formated: {row['description']} (must be 'Invitation ACTIVITY_NAME (NAME)')")

            activity_name = m.group(1)
            guest_name = m.group(2)

            if activity_name not in MAP_NAMEACTIVITY:
                raise IntegrityError(f"Activity {activity_name} is not found")
            activity = MAP_NAMEACTIVITY[activity_name]

            if guest_name not in MAP_NAMEGUEST:
                raise IntegrityError(f"Guest {guest_name} is not found")

            guest = None
            for g in MAP_NAMEGUEST[guest_name]:
                if g.activity.pk == activity.pk:
                    guest = g
                    break
            if guest is None:
                raise IntegrityError("Guest {guest_name} didn't go to the activity {activity_name}")

            obj_dict["guest"] = guest

            GuestTransaction.objects.get_or_create(**obj_dict)
        else:
            print("other type not supported yet:", ttype)


@transaction.atomic
def import_aliases(cur):
    cur.execute("SELECT * FROM aliases ORDER by id")
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["alias"])
        alias_name = row["alias"]
        alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
        obj_dict = {
            "note_id": MAP_IDBDE[row["idbde"]],
            "name": alias_name_good,
            "normalized_name": Alias.normalize(alias_name_good),
        }
        try:
            with transaction.atomic():
                alias, created = Alias.objects.get_or_create(**obj_dict)

        except IntegrityError as e:
            if "unique" in e.args[0]:
                continue
            else:
                raise e
        alias.save()


@transaction.atomic
def import_activities(cur):
    cur.execute("SELECT * FROM activites ORDER by id")
    n = cur.rowcount
    activity_type = ActivityType.objects.get(name="Pot")  # Need to be fixed manually
    kfet = Club.objects.get(name="Kfet")
    for idx, row in enumerate(cur):
        update_line(idx, n, row["alias"])
        organizer = Club.objects.filter(name=row["signature"])
        if organizer.exists():
            # Try to find the club that organizes the activity. If not founded, assume it's Kfet (fix manually)
            organizer = organizer.get()
        else:
            organizer = kfet
        obj_dict = {
            "name": row["titre"],
            "description": row["description"],
            "activity_type": activity_type,  # By default Pot
            "creater": MAP_IDBDE[row["responsable"]],
            "organizer": organizer,
            "attendees_club": kfet,  # Maybe fix manually
            "date_start": row["debut"],
            "date_end": row["fin"],
            "valid": row["validepar"] is not None,
            "open": row["open"],  # Should be always False
        }
        # WARNING: Fields lieu, liste, listeimprimee are missing
        try:
            with transaction.atomic():
                activity = Activity.objects.get_or_create(**obj_dict)[0]
                MAP_IDACTIVITY[row["id"]] = activity
                MAP_NAMEACTIVITY[activity.name] = activity
        except IntegrityError as e:
            raise e


@transaction.atomic
def import_activity_entries(cur):
    map_idguests = {}

    cur.execute("SELECT * FROM invites ORDER by id")
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["nom"] + " " + row["prenom"])
        obj_dict = {
            "activity": MAP_IDACTIVITY[row["activity"]],
            "last_name": row["nom"],
            "first_name": row["prenom"],
            "inviter": MAP_IDBDE[row["responsable"]],
        }
        try:
            with transaction.atomic():
                guest = Guest.objects.get_or_create(**obj_dict)[0]
                map_idguests.setdefault(row["responsable"], [])
                map_idguests[row["id"]].append(guest)
                guest_name = guest.first_name + " " + guest.last_name
                MAP_NAMEGUEST.setdefault(guest_name, [])
                MAP_NAMEGUEST[guest_name].append(guest)
        except IntegrityError as e:
            raise e

    cur.execute("SELECT * FROM entree_activites ORDER by id")
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["nom"] + " " + row["prenom"])
        activity = MAP_IDACTIVITY[row["activity"]]
        guest = None
        if row["est_invite"]:
            for g in map_idguests[row["id"]]:
                if g.activity.pk == activity.pk:
                    guest = g
                    break
            if not guest:
                raise IntegrityError("Guest was not found: " + str(row))
        obj_dict = {
            "activity": activity,
            "time": row["heure_entree"],
            "note": guest.inviter if guest else MAP_IDBDE[row["idbde"]],
            "guest": guest,
        }
        try:
            with transaction.atomic():
                Entry.objects.get_or_create(**obj_dict)
        except IntegrityError as e:
            raise e


@transaction.atomic
def import_remittances(cur):
    cur.execute("SELECT * FROM remises ORDER by id")
    map_idremittance = {}
    n = cur.rowcount
    check_type = RemittanceType.objects.get(note__name="Chèque")
    for idx, row in enumerate(cur):
        update_line(idx, n, row["date"])
        obj_dict = {
            "date": row["date"][10:],
            "remittance_type": check_type,
            "comment": row["commentaire"],
            "closed": row["close"],
        }
        try:
            with transaction.atomic():
                remittance = Remittance.objects.get_or_create(**obj_dict)
                map_idremittance[row["id"]] = remittance
        except IntegrityError as e:
            raise e

    print("remittances are imported")
    print("imported checks")

    cur.execute("SELECT * FROM cheques ORDER by id")
    n = cur.rowcount
    for idx, row in enumerate(cur):
        update_line(idx, n, row["date"])
        obj_dict = {
            "date": row["date"][10:],
            "remittance_type": check_type,
            "comment": row["commentaire"],
            "closed": row["close"],
        }
        tr = MAP_IDSPECIALTRANSACTION[row["idtransaction"]]
        proxy = SpecialTransactionProxy.objects.get_or_create(transaction=tr)
        proxy.remittance = map_idremittance[row["idremise"]]
        try:
            with transaction.atomic():
                proxy.save()
        except IntegrityError as e:
            raise e


class Command(BaseCommand):
    """
    Command for importing the database of NK15.
    Need to be run by a user with a registered role in postgres for the database nk15. 
    """

    def print_success(self, to_print):
        return self.stdout.write(self.style.SUCCESS(to_print))

    def add_arguments(self, parser):
        parser.add_argument('-c', '--comptes', action='store_true', help="import accounts")
        parser.add_argument('-b', '--boutons', action='store_true', help="import boutons")
        parser.add_argument('-t', '--transactions', action='store_true', help="import transaction")
        parser.add_argument('-al', '--aliases', action='store_true', help="import aliases")
        parser.add_argument('-ac', '--activities', action='store_true', help="import activities")
        parser.add_argument('-r', '--remittances', action='store_true', help="import check remittances")
        parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
        parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
        parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name')
        parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner')

    def handle(self, *args, **kwargs):
        global MAP_IDBDE
        nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
        # connecting to nk15 database
        conn = pg.connect(database=nk15db, user=nk15user)
        cur = conn.cursor(cursor_factory=pge.DictCursor)

        if kwargs["comptes"]:
            # reset database.
            call_command("migrate")
            call_command("loaddata", "initial")
            self.print_success("reset  nk20 database\n")
            import_comptes(cur)
            self.print_success("comptes table imported")
        elif kwargs["map"]:
            filename = kwargs["map"]
            with open(filename, 'r') as fp:
                MAP_IDBDE = json.load(fp)
                MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()}
        if kwargs["save"]:
            filename = kwargs["save"]
            with open(filename, 'w') as fp:
                json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)

        # /!\ need a prober MAP_IDBDE
        if kwargs["boutons"]:
            import_boutons(cur)
            self.print_success("boutons table imported\n")
        if kwargs["activities"]:
            import_activities(cur)
            self.print_success("activities imported\n")
            import_activity_entries(cur)
            self.print_success("activity entries imported\n")
        if kwargs["aliases"]:
            import_aliases(cur)
            self.print_success("aliases imported\n")
        if kwargs["transactions"]:
            import_transaction(cur)
            self.print_success("transaction imported\n")
        if kwargs["remittances"]:
            import_remittances(cur)
            self.print_success("remittances imported\n")