216 lines
7.6 KiB
Python
216 lines
7.6 KiB
Python
# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
|
|
# SPDX-License-Identifier: GPL-3.0-or-later
|
|
|
|
import psycopg2 as pg
|
|
import psycopg2.extras as pge
|
|
import datetime
|
|
import json
|
|
|
|
from django.utils.timezone import make_aware, now
|
|
from django.contrib.auth.models import User
|
|
from django.contrib.contenttypes.models import ContentType
|
|
|
|
from django.db import transaction
|
|
|
|
from note.models import Note, NoteUser, NoteClub
|
|
from note.models import Alias
|
|
from member.models import Club, Profile
|
|
|
|
from ._import_utils import ImportCommand, BulkCreateManager, timed
|
|
|
|
M_DURATION = 396
|
|
M_START = datetime.date(2019, 8, 31)
|
|
M_END = datetime.date(2020, 9, 30)
|
|
|
|
MAP_IDBDE = {
|
|
-4: 2, # Carte Bancaire
|
|
-3: 4, # Virement
|
|
-2: 1, # Especes
|
|
-1: 3, # Chèque
|
|
0: 5, # BDE
|
|
}
|
|
|
|
# some Aliases have been created in the fixtures
|
|
ALIAS_SET = {a[0] for a in Alias.objects.all().values_list("normalized_name")}
|
|
|
|
note_user_type = ContentType.objects.get(app_label="note", model="noteuser")
|
|
note_club_type = ContentType.objects.get(app_label="note", model="noteclub")
|
|
|
|
|
|
class Command(ImportCommand):
|
|
"""
|
|
Import command for People base data (Comptes, and Aliases)
|
|
"""
|
|
|
|
def add_arguments(self, parser):
|
|
parser.add_argument('-a', '--alias', action='store_true', help="import alias")
|
|
|
|
def import_special_account(self, cur):
|
|
cur.execute("SELECT idbde, solde from comptes where idbde <=0")
|
|
for row in cur:
|
|
note = Note.objects.get(pk=MAP_IDBDE[row["idbde"]])
|
|
note.amount = row["solde"]
|
|
note.save()
|
|
|
|
@timed
|
|
@transaction.atomic
|
|
def import_account(self, cur, chunk_size):
|
|
"""
|
|
Import every account of the nk15 in a batch fashion.
|
|
Every Model has to be manually created, and no magic `.save()`
|
|
function is being called.
|
|
"""
|
|
cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
|
|
pk_club = 3
|
|
pk_user = 1
|
|
pk_profile = 1
|
|
pk_note = 7 # pk 6 is Kfet!
|
|
n = cur.rowcount
|
|
|
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
|
for idx, row in enumerate(cur):
|
|
pseudo = row["pseudo"]
|
|
pseudo_norm = Alias.normalize(pseudo)
|
|
self.update_line(idx, n, pseudo)
|
|
# clean pseudo (normalized pseudo must be unique)
|
|
if pseudo_norm in ALIAS_SET:
|
|
pseudo = pseudo + str(row["idbde"])
|
|
else:
|
|
ALIAS_SET.add(pseudo_norm)
|
|
# clean date
|
|
note_dict = {
|
|
"pk": pk_note,
|
|
"balance": row['solde'],
|
|
"last_negative": None,
|
|
"is_active": True,
|
|
"display_image": "pic/default.png",
|
|
"created_at": now()
|
|
}
|
|
if row["last_negatif"] is not None:
|
|
note_dict["last_negative"] = make_aware(row["last_negatif"])
|
|
if row["type"] == "personne":
|
|
# sanitize password
|
|
if row["passwd"] != "*|*" and not row["deleted"]:
|
|
passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
|
|
else:
|
|
passwd_nk15 = ''
|
|
|
|
obj_dict = {
|
|
"pk": pk_user,
|
|
"username": row["pseudo"],
|
|
"password": passwd_nk15,
|
|
"first_name": row["nom"],
|
|
"last_name": row["prenom"],
|
|
"email": row["mail"],
|
|
"is_active": True, # temporary
|
|
}
|
|
profile_dict = {
|
|
"pk": pk_profile,
|
|
"user_id": pk_user,
|
|
"phone_number": row['tel'],
|
|
"address": row['adresse'],
|
|
"paid": row['normalien'],
|
|
"registration_valid": True,
|
|
"email_confirmed": True,
|
|
}
|
|
note_dict["polymorphic_ctype"] = note_user_type
|
|
note_user_dict = {
|
|
"pk": pk_note,
|
|
"user_id": pk_user,
|
|
}
|
|
alias_dict = {
|
|
"pk": pk_note,
|
|
"name": pseudo,
|
|
"normalized_name": Alias.normalize(pseudo),
|
|
"note_id": pk_note,
|
|
}
|
|
|
|
bulk_mgr.add(User(**obj_dict),
|
|
Profile(**profile_dict),
|
|
Note(**note_dict),
|
|
NoteUser(**note_user_dict),
|
|
Alias(**alias_dict),)
|
|
pk_user += 1
|
|
pk_profile += 1
|
|
|
|
else: # club
|
|
obj_dict = {
|
|
"pk": pk_club,
|
|
"name": row["pseudo"],
|
|
"email": row["mail"],
|
|
"membership_duration": M_DURATION,
|
|
"membership_start": M_START,
|
|
"membership_end": M_END,
|
|
"membership_fee_paid": 0,
|
|
"membership_fee_unpaid": 0,
|
|
}
|
|
note_club_dict = {
|
|
"pk": pk_note,
|
|
"club_id": pk_club,
|
|
}
|
|
alias_dict = {
|
|
"pk": pk_note,
|
|
"name": pseudo,
|
|
"normalized_name": Alias.normalize(pseudo),
|
|
"note_id": pk_note
|
|
}
|
|
note_dict["polymorphic_ctype"] = note_club_type
|
|
bulk_mgr.add(Club(**obj_dict),
|
|
Note(**note_dict),
|
|
NoteClub(**note_club_dict),
|
|
Alias(**alias_dict))
|
|
pk_club += 1
|
|
# row import completed
|
|
MAP_IDBDE[row["idbde"]] = pk_note
|
|
pk_note += 1
|
|
bulk_mgr.done()
|
|
|
|
@timed
|
|
def import_alias(self, cur, chunk_size):
|
|
"""
|
|
Import Alias from nk15
|
|
We rely on validation of the models, but it is slow.
|
|
"""
|
|
cur.execute("SELECT * FROM aliases ORDER by id")
|
|
n = cur.rowcount
|
|
|
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
|
pk_alias = Alias.objects.order_by('-id').first().id + 1
|
|
for idx, row in enumerate(cur):
|
|
alias_name = row["alias"]
|
|
alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
|
|
alias_norm = Alias.normalize(alias_name)
|
|
self.update_line(idx, n, alias_norm)
|
|
# clean pseudo (normalized pseudo must be unique)
|
|
if alias_norm in ALIAS_SET:
|
|
continue
|
|
else:
|
|
ALIAS_SET.add(alias_norm)
|
|
obj_dict = {
|
|
"pk": pk_alias,
|
|
"note_id": MAP_IDBDE[row["idbde"]],
|
|
"name": alias_name,
|
|
"normalized_name": alias_norm,
|
|
}
|
|
pk_alias += 1
|
|
bulk_mgr.add(Alias(**obj_dict))
|
|
bulk_mgr.done()
|
|
|
|
def handle(self, *args, **kwargs):
|
|
# default args, provided by ImportCommand.
|
|
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
|
|
# connecting to nk15 database
|
|
conn = pg.connect(database=nk15db, user=nk15user)
|
|
cur = conn.cursor(cursor_factory=pge.DictCursor)
|
|
|
|
self.import_special_account(cur)
|
|
self.import_account(cur, kwargs["chunk"])
|
|
# Alias Management
|
|
if kwargs["alias"]:
|
|
self.import_alias(cur, kwargs["chunk"])
|
|
# save to disk
|
|
if kwargs["save"]:
|
|
filename = kwargs["save"]
|
|
with open(filename, 'w') as fp:
|
|
json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
|