nk20-scripts/management/commands/import_account.py

#!/usr/bin/env python3

import psycopg2 as pg
import psycopg2.extras as pge
import datetime

from django.utils.timezone import make_aware, now
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType

from django.db import transaction
from django.db import IntegrityError

from note.models import Note, NoteUser, NoteClub
from note.models import Alias
from member.models import Club, Profile

from ._import_utils import ImportCommand, BulkCreateManager

M_DURATION = 396
M_START = datetime.date(2019, 8, 31)
M_END = datetime.date(2020, 9, 30)

MAP_IDBDE = {
    -4: 2,  # Carte Bancaire
    -3: 4,  # Virement
    -2: 1,  # Especes
    -1: 3,  # Chèque
    0: 5,  # BDE
}
note_user_type = ContentType.objects.get(app_label="note", model="noteuser")
note_club_type = ContentType.objects.get(app_label="note", model="noteclub")


class Command(ImportCommand):
    """
    Import command for People base data (Comptes, and Aliases)
    """
   
    def add_arguments(self, parser):
        parser.add_argument('-a', '--alias', action='store_true', help="import alias")
      
    @transaction.atomic
    def import_account(self, cur, chunk_size):
        """
        Import every account of the nk15 in a batch fashion.
        Every Model has to be manually created, and no magic `.save()`
        function is being called.
        """
        cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
        pk_club = 3
        pk_user = 1
        pk_profile = 1
        pk_note = 7 # pk 6 is Kfet!
        n = cur.rowcount

        bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
        pseudo_list = set()
        for idx, row in enumerate(cur):
            pseudo = row["pseudo"]
            pseudo_norm = Alias.normalize(pseudo)
            self.update_line(idx, n, pseudo)
            # clean pseudo (normalized pseudo must be unique)
            if pseudo_norm in pseudo_list:
                pseudo = pseudo+str(row["idbde"])
            else:
                pseudo_list.add(pseudo_norm)
            # clean date
            note_dict = {
                "pk": pk_note,
                "balance": 0,
                "last_negative": None,
                "is_active": True,
                "display_image": "",
                "created_at": now()
            }
            if row["last_negatif"] is not None:
                note_dict["last_negative"] = make_aware(row["last_negatif"])
            if row["type"] == "personne":
                # sanitize password
                if row["passwd"] != "*|*" and not row["deleted"]:
                    passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
                else:
                    passwd_nk15 = ''

                obj_dict = {
                    "pk": pk_user,
                    "username": row["pseudo"],
                    "password": passwd_nk15,
                    "first_name": row["nom"],
                    "last_name": row["prenom"],
                    "email": row["mail"],
                    "is_active": True,  # temporary
                }
                profile_dict = {
                    "pk": pk_profile,
                    "user_id": pk_user,
                    "phone_number": row['tel'],
                    "address": row['adresse'],
                    "paid": row['normalien'],
                    "registration_valid": True,
                    "email_confirmed": True,
                }
                note_dict["polymorphic_ctype"] = note_user_type
                note_user_dict = {
                    "pk": pk_note,
                    "user_id": pk_user,
                }
                alias_dict = {
                    "pk": pk_note,
                    "name": pseudo,
                    "normalized_name": Alias.normalize(pseudo),
                    "note_id": pk_note,
                }

                bulk_mgr.add(User(**obj_dict),
                             Profile(**profile_dict),
                             Note(**note_dict),
                             NoteUser(**note_user_dict),
                             Alias(**alias_dict),)
                pk_user += 1
                pk_profile += 1

            else:  # club
                obj_dict = {
                    "pk": pk_club,
                    "name": row["pseudo"],
                    "email": row["mail"],
                    "membership_duration": M_DURATION,
                    "membership_start": M_START,
                    "membership_end": M_END,
                    "membership_fee_paid": 0,
                    "membership_fee_unpaid": 0,
                }
                note_club_dict = {
                    "pk": pk_note,
                    "club_id": pk_club,
                }
                alias_dict = {
                    "pk": pk_note,
                    "name": pseudo,
                    "normalized_name": Alias.normalize(pseudo),
                    "note_id": pk_note
                }
                note_dict["polymorphic_ctype"] = note_club_type
                bulk_mgr.add(Club(**obj_dict),
                             Note(**note_dict),
                             NoteClub(**note_club_dict),
                             Alias(**alias_dict))
                pk_club += 1
            # row import completed
            MAP_IDBDE[row["idbde"]] = pk_note
            pk_note += 1
        self.print_success("comptes table imported")

    def import_alias(self, cur):
        """
        Import Alias from nk15
        We rely on validation of the models, but it is slow.
        """
        cur.execute("SELECT * FROM aliases ORDER by id")
        n = cur.rowcount
        for idx, row in enumerate(cur):
            self.update_line(idx, n, row["alias"])
            alias_name = row["alias"]
            alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
            obj_dict = {
                "note_id": MAP_IDBDE[row["idbde"]],
                "name": alias_name_good,
                "normalized_name": Alias.normalize(alias_name_good),
            }
            try:
                with transaction.atomic():
                    alias, created = Alias.objects.get_or_create(**obj_dict)
            except IntegrityError as e:
                if "unique" in e.args[0]:
                    continue
                else:
                    raise e
            alias.save()
        return None

    def handle(self, *args, **kwargs):
        global MAP_IDBDE
        # default args, provided by ImportCommand.
        nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
        # connecting to nk15 database
        conn = pg.connect(database=nk15db, user=nk15user)
        cur = conn.cursor(cursor_factory=pge.DictCursor)

        self.import_account(cur,kwargs["chunk"])
        if kwargs["save"]:
            filename = kwargs["save"]
            with open(filename, 'w') as fp:
                json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
        # Alias Management
        if kwargs["alias"]:
            self.impot_alias(cur)