258 lines
10 KiB
Python
258 lines
10 KiB
Python
# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
import psycopg2 as pg
|
|
import psycopg2.extras as pge
|
|
import datetime
|
|
import json
|
|
|
|
from django.template.loader import render_to_string
|
|
from django.utils.timezone import make_aware, now
|
|
from django.contrib.auth.models import User
|
|
from django.contrib.contenttypes.models import ContentType
|
|
|
|
from django.db import transaction
|
|
|
|
from note.models import Note, NoteUser, NoteClub
|
|
from note.models import Alias
|
|
from member.models import Club, Profile
|
|
|
|
from ._import_utils import ImportCommand, BulkCreateManager, timed
|
|
|
|
M_DURATION = 396
|
|
M_START = datetime.date(2019, 8, 1)
|
|
M_END = datetime.date(2020, 9, 30)
|
|
|
|
MAP_IDBDE = {
|
|
-4: 2, # Carte Bancaire
|
|
-3: 4, # Virement
|
|
-2: 1, # Especes
|
|
-1: 3, # Chèque
|
|
0: 5, # BDE
|
|
}
|
|
|
|
# some Aliases have been created in the fixtures
|
|
ALIAS_SET = {a[0] for a in Alias.objects.all().values_list("normalized_name")}
|
|
|
|
# Some people might loose some aliases due to normalization. We warn them on them.
|
|
LOST_ALIASES = {}
|
|
# In some rare cases, the username might be in conflict with some others. We change them and warn the users.
|
|
CHANGED_USERNAMES = []
|
|
|
|
note_user_type = ContentType.objects.get(app_label="note", model="noteuser")
|
|
note_club_type = ContentType.objects.get(app_label="note", model="noteclub")
|
|
|
|
|
|
class Command(ImportCommand):
|
|
"""
|
|
Import command for People base data (Comptes, and Aliases)
|
|
"""
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('-a', '--alias', action='store_true', help="import alias")
|
|
|
|
def import_special_account(self, cur):
|
|
cur.execute("SELECT idbde, solde from comptes where idbde <=0")
|
|
for row in cur:
|
|
note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]])
|
|
note.amount = row["solde"]
|
|
note.save()
|
|
|
|
@timed
|
|
@transaction.atomic
|
|
def import_account(self, cur, chunk_size):
|
|
"""
|
|
Import every account of the nk15 in a batch fashion.
|
|
Every Model has to be manually created, and no magic `.save()`
|
|
function is being called.
|
|
"""
|
|
# Get promotion and date of creation of the account
|
|
cur.execute("SELECT idbde, MIN(date) AS created_at, MIN(annee) AS promo FROM adhesions"
|
|
" GROUP BY idbde ORDER BY promo, created_at;")
|
|
MAP_IDBDE_PROMOTION = {}
|
|
for row in cur:
|
|
MAP_IDBDE_PROMOTION[row["idbde"]] = row
|
|
|
|
cur.execute("SELECT * FROM comptes WHERE idbde <= 0 ORDER BY idbde;")
|
|
for row in cur:
|
|
note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]])
|
|
note.balance = row["solde"]
|
|
note.save()
|
|
|
|
cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
|
|
pk_club = 3
|
|
pk_user = 1
|
|
pk_profile = 1
|
|
pk_note = 7 # pk 6 is Kfet!
|
|
n = cur.rowcount
|
|
|
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
|
for idx, row in enumerate(cur):
|
|
pseudo = row["pseudo"]
|
|
pseudo_norm = Alias.normalize(pseudo)
|
|
self.update_line(idx, n, pseudo)
|
|
# clean pseudo (normalized pseudo must be unique and not empty)
|
|
if not pseudo_norm or pseudo_norm in ALIAS_SET:
|
|
pseudo = pseudo + str(row["idbde"])
|
|
CHANGED_USERNAMES.append((pk_note, row["idbde"], pseudo))
|
|
else:
|
|
ALIAS_SET.add(pseudo_norm)
|
|
# clean date
|
|
note_dict = {
|
|
"pk": pk_note,
|
|
"balance": row['solde'],
|
|
"last_negative": None,
|
|
"is_active": not row["bloque"],
|
|
"display_image": "pic/default.png",
|
|
}
|
|
if row["last_negatif"] is not None:
|
|
note_dict["last_negative"] = make_aware(row["last_negatif"])
|
|
if row["type"] == "personne":
|
|
# sanitize password
|
|
if row["passwd"] != "*|*" and not row["deleted"]:
|
|
passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
|
|
else:
|
|
passwd_nk15 = ''
|
|
|
|
# Note account should have no password and be active
|
|
if int(row["idbde"]) == 3508:
|
|
passwd_nk15 = "ipbased$127.0.0.1"
|
|
row["bloque"] = False
|
|
|
|
if row["idbde"] not in MAP_IDBDE_PROMOTION:
|
|
# NK12 bug. Applying default values
|
|
MAP_IDBDE_PROMOTION[row["idbde"]] = {"promo": 2014,
|
|
"created_at": datetime.datetime(2014, 9, 1, 0, 0, 0)}
|
|
|
|
obj_dict = {
|
|
"pk": pk_user,
|
|
"username": row["pseudo"],
|
|
"password": passwd_nk15,
|
|
"first_name": row["prenom"],
|
|
"last_name": row["nom"],
|
|
"email": row["mail"],
|
|
"is_active": not row["bloque"],
|
|
"date_joined": make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"]),
|
|
}
|
|
profile_dict = {
|
|
"pk": pk_profile,
|
|
"user_id": pk_user,
|
|
"phone_number": row['tel'],
|
|
"address": row['adresse'],
|
|
"paid": row['normalien'],
|
|
"registration_valid": True,
|
|
"email_confirmed": True,
|
|
"promotion": MAP_IDBDE_PROMOTION[row["idbde"]]["promo"],
|
|
"report_frequency": max(row["report_period"], 0),
|
|
"last_report": make_aware(row["previous_report_date"]),
|
|
}
|
|
note_dict["created_at"] = make_aware(MAP_IDBDE_PROMOTION[row["idbde"]]["created_at"])
|
|
note_dict["polymorphic_ctype"] = note_user_type
|
|
note_user_dict = {
|
|
"pk": pk_note,
|
|
"user_id": pk_user,
|
|
}
|
|
alias_dict = {
|
|
"pk": pk_note,
|
|
"name": pseudo,
|
|
"normalized_name": Alias.normalize(pseudo),
|
|
"note_id": pk_note,
|
|
}
|
|
|
|
bulk_mgr.add(User(**obj_dict),
|
|
Profile(**profile_dict),
|
|
Note(**note_dict),
|
|
NoteUser(**note_user_dict),
|
|
Alias(**alias_dict),)
|
|
pk_user += 1
|
|
pk_profile += 1
|
|
|
|
else: # club
|
|
obj_dict = {
|
|
"pk": pk_club,
|
|
"name": row["pseudo"],
|
|
"email": row["mail"],
|
|
"parent_club_id": 1, # All clubs depends on BDE by default
|
|
"membership_duration": M_DURATION,
|
|
"membership_start": M_START,
|
|
"membership_end": M_END,
|
|
"membership_fee_paid": 0,
|
|
"membership_fee_unpaid": 0,
|
|
}
|
|
note_club_dict = {
|
|
"pk": pk_note,
|
|
"club_id": pk_club
|
|
}
|
|
alias_dict = {
|
|
"pk": pk_note,
|
|
"name": pseudo,
|
|
"normalized_name": Alias.normalize(pseudo),
|
|
"note_id": pk_note
|
|
}
|
|
note_dict["created_at"] = make_aware(row["previous_report_date"]) # Not perfect, but good approximation
|
|
note_dict["polymorphic_ctype"] = note_club_type
|
|
bulk_mgr.add(Club(**obj_dict),
|
|
Note(**note_dict),
|
|
NoteClub(**note_club_dict),
|
|
Alias(**alias_dict))
|
|
pk_club += 1
|
|
# row import completed
|
|
MAP_IDBDE[row["idbde"]] = pk_note
|
|
pk_note += 1
|
|
bulk_mgr.done()
|
|
|
|
@timed
|
|
def import_alias(self, cur, chunk_size):
|
|
"""
|
|
Import Alias from nk15
|
|
We rely on validation of the models, but it is slow.
|
|
"""
|
|
cur.execute("SELECT * FROM aliases ORDER by id")
|
|
n = cur.rowcount
|
|
|
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
|
pk_alias = Alias.objects.order_by('-id').first().id + 1
|
|
for idx, row in enumerate(cur):
|
|
alias_name = row["alias"]
|
|
alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
|
|
alias_norm = Alias.normalize(alias_name)
|
|
self.update_line(idx, n, alias_norm)
|
|
# clean pseudo (normalized pseudo must be unique)
|
|
if not alias_norm or alias_norm in ALIAS_SET:
|
|
LOST_ALIASES.setdefault(MAP_IDBDE[row["idbde"]], [])
|
|
LOST_ALIASES[MAP_IDBDE[row["idbde"]]].append(alias_name)
|
|
continue
|
|
else:
|
|
ALIAS_SET.add(alias_norm)
|
|
obj_dict = {
|
|
"pk": pk_alias,
|
|
"note_id": MAP_IDBDE[row["idbde"]],
|
|
"name": alias_name,
|
|
"normalized_name": alias_norm,
|
|
}
|
|
pk_alias += 1
|
|
bulk_mgr.add(Alias(**obj_dict))
|
|
bulk_mgr.done()
|
|
|
|
def handle(self, *args, **kwargs):
|
|
# default args, provided by ImportCommand.
|
|
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
|
|
# connecting to nk15 database
|
|
conn = pg.connect(database=nk15db, user=nk15user)
|
|
cur = conn.cursor(cursor_factory=pge.DictCursor)
|
|
|
|
self.import_special_account(cur)
|
|
self.import_account(cur, kwargs["chunk"])
|
|
# Alias Management
|
|
if kwargs["alias"]:
|
|
self.import_alias(cur, kwargs["chunk"])
|
|
# save to disk
|
|
if kwargs["save"]:
|
|
filename = kwargs["save"]
|
|
with open(filename, 'w') as fp:
|
|
json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
|
|
with open(filename + ".changed-usernames", 'w') as fp:
|
|
json.dump(CHANGED_USERNAMES, fp, sort_keys=True, indent=2)
|
|
with open(filename + ".removed-aliases", 'w') as fp:
|
|
json.dump(LOST_ALIASES, fp, sort_keys=True, indent=2)
|