# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay # SPDX-License-Identifier: GPL-3.0-or-later import psycopg2 as pg import psycopg2.extras as pge import datetime import json from django.template.loader import render_to_string from django.utils.timezone import make_aware, now from django.contrib.auth.models import User from django.contrib.contenttypes.models import ContentType from django.db import transaction from note.models import Note, NoteUser, NoteClub from note.models import Alias from member.models import Club, Profile from ._import_utils import ImportCommand, BulkCreateManager, timed M_DURATION = 396 M_START = datetime.date(2019, 8, 1) M_END = datetime.date(2020, 9, 30) MAP_IDBDE = { -4: 2, # Carte Bancaire -3: 4, # Virement -2: 1, # Especes -1: 3, # Chèque 0: 5, # BDE } # some Aliases have been created in the fixtures ALIAS_SET = {a[0] for a in Alias.objects.all().values_list("normalized_name")} # Some people might loose some aliases due to normalization. We warn them on them. LOST_ALIASES = {} # In some rare cases, the username might be in conflict with some others. We change them and warn the users. CHANGED_USERNAMES = [] note_user_type = ContentType.objects.get(app_label="note", model="noteuser") note_club_type = ContentType.objects.get(app_label="note", model="noteclub") class Command(ImportCommand): """ Import command for People base data (Comptes, and Aliases) """ def add_arguments(self, parser): parser.add_argument('-a', '--alias', action='store_true', help="import alias") def import_special_account(self, cur): cur.execute("SELECT idbde, solde from comptes where idbde <=0") for row in cur: note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]]) note.amount = row["solde"] note.save() @timed @transaction.atomic def import_account(self, cur, chunk_size): """ Import every account of the nk15 in a batch fashion. Every Model has to be manually created, and no magic `.save()` function is being called. """ # Get promotion and date of creation of the account cur.execute("SELECT idbde, MIN(date) AS created_at, MIN(annee) AS promo FROM adhesions" " GROUP BY idbde ORDER BY promo, created_at;") MAP_IDBDE_PROMOTION = {} for row in cur: MAP_IDBDE_PROMOTION[row["idbde"]] = row cur.execute("SELECT * FROM comptes WHERE idbde <= 0 ORDER BY idbde;") for row in cur: note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]]) note.balance = row["solde"] note.save() cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;") pk_club = 3 pk_user = 1 pk_profile = 1 pk_note = 7 # pk 6 is Kfet! n = cur.rowcount bulk_mgr = BulkCreateManager(chunk_size=chunk_size) for idx, row in enumerate(cur): pseudo = row["pseudo"] pseudo_norm = Alias.normalize(pseudo) self.update_line(idx, n, pseudo) # clean pseudo (normalized pseudo must be unique and not empty) if not pseudo_norm or pseudo_norm in ALIAS_SET: pseudo = pseudo + str(row["idbde"]) CHANGED_USERNAMES.append((pk_note, row["idbde"], pseudo)) else: ALIAS_SET.add(pseudo_norm) # clean date note_dict = { "pk": pk_note, "balance": row['solde'], "last_negative": None, "is_active": not row["bloque"], "display_image": "pic/default.png", } if row["last_negatif"] is not None: note_dict["last_negative"] = make_aware(row["last_negatif"]) if row["type"] == "personne": # sanitize password if row["passwd"] != "*|*" and not row["deleted"]: passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]]) else: passwd_nk15 = '' # Note account should have no password and be active if int(row["idbde"]) == 3508: passwd_nk15 = "ipbased$127.0.0.1" row["bloque"] = False if row["idbde"] not in MAP_IDBDE_PROMOTION: # NK12 bug. Applying default values MAP_IDBDE_PROMOTION[row["idbde"]] = {"promo": 2014, "created_at": datetime.datetime(2014, 9, 1, 0, 0, 0)} obj_dict = { "pk": pk_user, "username": row["pseudo"], "password": passwd_nk15, "first_name": row["prenom"], "last_name": row["nom"], "email": row["mail"], "is_active": not row["bloque"], "date_joined": make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"]), } profile_dict = { "pk": pk_profile, "user_id": pk_user, "phone_number": row['tel'], "address": row['adresse'], "paid": row['normalien'], "registration_valid": True, "email_confirmed": True, "promotion": MAP_IDBDE_PROMOTION[row["idbde"]]["promo"], "report_frequency": max(row["report_period"], 0), "last_report": make_aware(row["previous_report_date"]), } note_dict["created_at"] = make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"]) note_dict["polymorphic_ctype"] = note_user_type note_user_dict = { "pk": pk_note, "user_id": pk_user, } alias_dict = { "pk": pk_note, "name": pseudo, "normalized_name": Alias.normalize(pseudo), "note_id": pk_note, } bulk_mgr.add(User(**obj_dict), Profile(**profile_dict), Note(**note_dict), NoteUser(**note_user_dict), Alias(**alias_dict),) pk_user += 1 pk_profile += 1 else: # club obj_dict = { "pk": pk_club, "name": row["pseudo"], "email": row["mail"], "parent_club_id": 1, # All clubs depends on BDE by default "membership_duration": M_DURATION, "membership_start": M_START, "membership_end": M_END, "membership_fee_paid": 0, "membership_fee_unpaid": 0, } note_club_dict = { "pk": pk_note, "club_id": pk_club } alias_dict = { "pk": pk_note, "name": pseudo, "normalized_name": Alias.normalize(pseudo), "note_id": pk_note } note_dict["created_at"] = make_aware(row["previous_report_date"]) # Not perfect, but good approximation note_dict["polymorphic_ctype"] = note_club_type bulk_mgr.add(Club(**obj_dict), Note(**note_dict), NoteClub(**note_club_dict), Alias(**alias_dict)) pk_club += 1 # row import completed MAP_IDBDE[row["idbde"]] = pk_note pk_note += 1 bulk_mgr.done() @timed def import_alias(self, cur, chunk_size): """ Import Alias from nk15 We rely on validation of the models, but it is slow. """ cur.execute("SELECT * FROM aliases ORDER by id") n = cur.rowcount bulk_mgr = BulkCreateManager(chunk_size=chunk_size) pk_alias = Alias.objects.order_by('-id').first().id + 1 for idx, row in enumerate(cur): alias_name = row["alias"] alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name alias_norm = Alias.normalize(alias_name) self.update_line(idx, n, alias_norm) # clean pseudo (normalized pseudo must be unique) if not alias_norm or alias_norm in ALIAS_SET: LOST_ALIASES.setdefault(MAP_IDBDE[row["idbde"]], []) LOST_ALIASES[MAP_IDBDE[row["idbde"]]].append(alias_name) continue else: ALIAS_SET.add(alias_norm) obj_dict = { "pk": pk_alias, "note_id": MAP_IDBDE[row["idbde"]], "name": alias_name, "normalized_name": alias_norm, } pk_alias += 1 bulk_mgr.add(Alias(**obj_dict)) bulk_mgr.done() def handle(self, *args, **kwargs): # default args, provided by ImportCommand. nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] # connecting to nk15 database conn = pg.connect(database=nk15db, user=nk15user) cur = conn.cursor(cursor_factory=pge.DictCursor) self.import_special_account(cur) self.import_account(cur, kwargs["chunk"]) # Alias Management if kwargs["alias"]: self.import_alias(cur, kwargs["chunk"]) # save to disk if kwargs["save"]: filename = kwargs["save"] with open(filename, 'w') as fp: json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2) with open(filename + ".changed-usernames", 'w') as fp: json.dump(CHANGED_USERNAMES, fp, sort_keys=True, indent=2) with open(filename + ".removed-aliases", 'w') as fp: json.dump(LOST_ALIASES, fp, sort_keys=True, indent=2)