nk20-scripts/management/commands/import_nk15.py

520 lines
19 KiB
Python

#!/usr/env/bin python3
import json
import datetime
import re
import pytz
import psycopg2 as pg
import psycopg2.extras as pge
from django.core.management.base import BaseCommand
from django.core.management import call_command
from django.db import transaction
from django.core.exceptions import ValidationError
from django.utils.timezone import make_aware
from django.db import IntegrityError
from django.contrib.auth.models import User
from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction
from note.models import Note
from note.models import Alias
from note.models import (
TemplateCategory,
TransactionTemplate,
Transaction,
RecurrentTransaction,
MembershipTransaction,
SpecialTransaction,
)
from member.models import Club, Membership
from treasury.models import RemittanceType, Remittance, SpecialTransactionProxy
"""
Script d'import de la nk15:
TODO: import transactions
TODO: import adhesion
TODO: import activite
TODO: import ...
"""
M_DURATION = 396
M_START = datetime.date(2019, 8, 31)
M_END = datetime.date(2020, 9, 30)
MAP_IDBDE = {
-4: 2, # Carte Bancaire
-3: 4, # Virement
-2: 1, # Especes
-1: 3, # Chèque
0: 5, # BDE
}
MAP_IDACTIVITY = {}
MAP_NAMEACTIVITY = {}
MAP_NAMEGUEST = {}
MAP_IDSPECIALTRANSACTION = {}
def update_line(n, total, content):
n = str(n)
total = str(total)
n.rjust(len(total))
print(f"\r ({n}/{total}) {content:10.10}", end="")
@transaction.atomic
def import_comptes(cur):
cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
pkclub = 3
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["pseudo"])
if row["type"] == "personne":
# sanitize password
if row["passwd"] != "*|*" and not row["deleted"]:
passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
else:
passwd_nk15 = ''
try:
obj_dict = {
"username": row["pseudo"],
"password": passwd_nk15,
"first_name": row["nom"],
"last_name": row["prenom"],
"email": row["mail"],
"is_active": True, # temporary
}
user = User.objects.create(**obj_dict)
profile = user.profile
profile.phone_number = row['tel']
profile.address = row['adresse']
profile.paid = row['normalien']
profile.registration_valid = True
profile.email_confirmed = True
user.save()
profile.save()
# sanitize duplicate aliases (nk12)
except ValidationError as e:
if e.code == 'same_alias':
user.username = row["pseudo"] + str(row["idbde"])
user.save()
else:
raise e
# profile and note created via signal.
note = user.note
date = row.get("last_negatif", None)
if date is not None:
note.last_negative = make_aware(date)
note.balance = row["solde"]
note.save()
else: # club
obj_dict = {
"pk": pkclub,
"name": row["pseudo"],
"email": row["mail"],
"membership_duration": M_DURATION,
"membership_start": M_START,
"membership_end": M_END,
"membership_fee_paid": 0,
"membership_fee_unpaid": 0,
}
club, c = Club.objects.get_or_create(**obj_dict)
pkclub += 1
note = club.note
note.balance = row["solde"]
club.save()
note.save()
MAP_IDBDE[row["idbde"]] = note.note_ptr_id
@transaction.atomic
def import_boutons(cur):
cur.execute("SELECT * FROM boutons;")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["label"])
cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"])
if created:
cat.save()
obj_dict = {
"pk": row["id"],
"name": row["label"],
"amount": row["montant"],
"destination_id": MAP_IDBDE[row["destinataire"]],
"category": cat,
"display": row["affiche"],
"description": row["description"],
}
try:
with transaction.atomic(): # required for error management
button = TransactionTemplate.objects.create(**obj_dict)
except IntegrityError as e:
# button with the same name is not possible in NK20.
if "unique" in e.args[0]:
qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name')
note_name = qs[0]["name"]
# rename button name
obj_dict["name"] = f"{obj_dict_name['name']} {note_name}"
button = TransactionTemplate.objects.create(**obj_dict)
else:
raise e
button.save()
@transaction.atomic
def import_transaction(cur):
idmin = 58770
bde = Club.objects.get(name="BDE")
kfet = Club.objects.get(name="Kfet")
cur.execute(
"SELECT t.date AS transac_date, t.type, t.emetteur,\
t.destinataire,t.quantite, t.montant, t.description,\
t.valide, t.cantinvalidate, t.categorie, \
a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
FROM transactions AS t \
LEFT JOIN adhesions AS a ON t.id = a.idtransaction \
WHERE t.id> {} \
ORDER BY t.id;".format(idmin)
)
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["description"])
try:
date = make_aware(row["transac_date"])
except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))
# standart transaction object
obj_dict = {
# "pk": row["id"],
"destination_id": MAP_IDBDE[row["destinataire"]],
"source_id": MAP_IDBDE[row["emetteur"]],
"created_at": date,
"amount": row["montant"],
"quantity": row["quantite"],
"reason": row["description"],
"valid": row["valide"],
}
ttype = row["type"]
if ttype == "don" or ttype == "transfert":
Transaction.objects.create(**obj_dict)
elif ttype == "bouton":
cat_name = row["categorie"]
if cat_name is None:
cat_name = 'None'
cat, created = TemplateCategory.objects.get_or_create(name=cat_name)
if created:
cat.save()
obj_dict["category"] = cat
RecurrentTransaction.objects.create(**obj_dict)
elif ttype == "crédit" or ttype == "retrait":
field_id = "source_id" if ttype == "crédit" else "destination_id"
if "espèce" in row["description"]:
obj_dict[field_id] = 1
elif "carte" in row["description"]:
obj_dict[field_id] = 2
elif "cheques" in row["description"]:
obj_dict[field_id] = 3
elif "virement" in row["description"]:
obj_dict[field_id] = 4
pk = max(row["destinataire"], row["emetteur"])
actor = Note.objects.get(id=MAP_IDBDE[pk])
# custom fields of SpecialTransaction
if actor.__class__.__name__ == "NoteUser":
obj_dict["first_name"] = actor.user.first_name
obj_dict["last_name"] = actor.user.last_name
elif actor.__class__.__name__ == "NoteClub":
obj_dict["first_name"] = actor.club.name
obj_dict["last_name"] = actor.club.name
else:
raise Exception("Badly formatted Special Transaction You should'nt be there.")
tr = SpecialTransaction.objects.create(**obj_dict)
if "cheques" in row["description"]:
MAP_IDSPECIALTRANSACTION[row["id"]] = tr
elif ttype == "adhésion":
montant = row["montant"]
# Create Double membership to Kfet and Bde
# sometimes montant = 0, fees are modified accordingly.
bde_dict = {
"user": MAP_IDBDE[row["idbde"]],
"club": bde,
"date_start": row["date"].date(), # Only date, not time
"fee": min(500, montant)
}
kfet_dict = {
"user": MAP_IDBDE[row["idbde"]],
"club": kfet,
"date_start": row["date"].date(), # Only date, not time
"fee": max(montant - 500, 0),
}
if row["valide"]:
with transaction.atomic():
# membership save triggers MembershipTransaction creation
bde_membership = Membership.objects.get_or_create(**bde_dict)
kfet_membership = Membership.objects.get_or_create(**kfet_dict)
bde_membership.transaction.created_at = row["transac_date"]
bde_membership.transaction.description = row["description"]
bde_membership.transaction.save()
kfet_membership.transaction.created_at = row["transac_date"]
kfet_membership.transaction.description = row["description"] + "(Kfet)"
kfet_membership.transaction.save()
else:
# don't create membership
MembershipTransaction.objects.create(**obj_dict)
elif ttype == "invitation":
m = re.search(r"Invitation (.*?) \((.*?)\)", row["description"])
if m is None:
raise IntegrityError(f"Invitation is not well formated: {row['description']} (must be 'Invitation ACTIVITY_NAME (NAME)')")
activity_name = m.group(1)
guest_name = m.group(2)
if activity_name not in MAP_NAMEACTIVITY:
raise IntegrityError(f"Activity {activity_name} is not found")
activity = MAP_NAMEACTIVITY[activity_name]
if guest_name not in MAP_NAMEGUEST:
raise IntegrityError(f"Guest {guest_name} is not found")
guest = None
for g in MAP_NAMEGUEST[guest_name]:
if g.activity.pk == activity.pk:
guest = g
break
if guest is None:
raise IntegrityError("Guest {guest_name} didn't go to the activity {activity_name}")
obj_dict["guest"] = guest
GuestTransaction.objects.get_or_create(**obj_dict)
else:
print("other type not supported yet:", ttype)
@transaction.atomic
def import_aliases(cur):
cur.execute("SELECT * FROM aliases ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["alias"])
alias_name = row["alias"]
alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
obj_dict = {
"note_id": MAP_IDBDE[row["idbde"]],
"name": alias_name_good,
"normalized_name": Alias.normalize(alias_name_good),
}
try:
with transaction.atomic():
alias, created = Alias.objects.get_or_create(**obj_dict)
except IntegrityError as e:
if "unique" in e.args[0]:
continue
else:
raise e
alias.save()
@transaction.atomic
def import_activities(cur):
cur.execute("SELECT * FROM activites ORDER by id")
n = cur.rowcount
activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually
kfet = Club.objects.get(name="Kfet")
for idx, row in enumerate(cur):
update_line(idx, n, row["alias"])
organizer = Club.objects.filter(name=row["signature"])
if organizer.exists():
# Try to find the club that organizes the activity. If not found, assume it's Kfet (fix manually)
organizer = organizer.get()
else:
organizer = kfet
obj_dict = {
"name": row["titre"],
"description": row["description"],
"activity_type": activity_type, # By default Pot
"creater": MAP_IDBDE[row["responsable"]],
"organizer": organizer,
"attendees_club": kfet, # Maybe fix manually
"date_start": row["debut"],
"date_end": row["fin"],
"valid": row["validepar"] is not None,
"open": row["open"], # Should always be False
}
# WARNING: Fields lieu, liste, listeimprimee are missing
try:
with transaction.atomic():
activity = Activity.objects.get_or_create(**obj_dict)[0]
MAP_IDACTIVITY[row["id"]] = activity
MAP_NAMEACTIVITY[activity.name] = activity
except IntegrityError as e:
raise e
@transaction.atomic
def import_activity_entries(cur):
map_idguests = {}
cur.execute("SELECT * FROM invites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
obj_dict = {
"activity": MAP_IDACTIVITY[row["activity"]],
"last_name": row["nom"],
"first_name": row["prenom"],
"inviter": MAP_IDBDE[row["responsable"]],
}
try:
with transaction.atomic():
guest = Guest.objects.get_or_create(**obj_dict)[0]
map_idguests.setdefault(row["responsable"], [])
map_idguests[row["id"]].append(guest)
guest_name = guest.first_name + " " + guest.last_name
MAP_NAMEGUEST.setdefault(guest_name, [])
MAP_NAMEGUEST[guest_name].append(guest)
except IntegrityError as e:
raise e
cur.execute("SELECT * FROM entree_activites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
activity = MAP_IDACTIVITY[row["activity"]]
guest = None
if row["est_invite"]:
for g in map_idguests[row["id"]]:
if g.activity.pk == activity.pk:
guest = g
break
if not guest:
raise IntegrityError("Guest was not found: " + str(row))
obj_dict = {
"activity": activity,
"time": row["heure_entree"],
"note": guest.inviter if guest else MAP_IDBDE[row["idbde"]],
"guest": guest,
}
try:
with transaction.atomic():
Entry.objects.get_or_create(**obj_dict)
except IntegrityError as e:
raise e
@transaction.atomic
def import_remittances(cur):
cur.execute("SELECT * FROM remises ORDER by id")
map_idremittance = {}
n = cur.rowcount
check_type = RemittanceType.objects.get(note__name="Chèque")
for idx, row in enumerate(cur):
update_line(idx, n, row["date"])
obj_dict = {
"date": row["date"][10:],
"remittance_type": check_type,
"comment": row["commentaire"],
"closed": row["close"],
}
try:
with transaction.atomic():
remittance = Remittance.objects.get_or_create(**obj_dict)
map_idremittance[row["id"]] = remittance
except IntegrityError as e:
raise e
print("remittances are imported")
print("imported checks")
cur.execute("SELECT * FROM cheques ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["date"])
obj_dict = {
"date": row["date"][10:],
"remittance_type": check_type,
"comment": row["commentaire"],
"closed": row["close"],
}
tr = MAP_IDSPECIALTRANSACTION[row["idtransaction"]]
proxy = SpecialTransactionProxy.objects.get_or_create(transaction=tr)
proxy.remittance = map_idremittance[row["idremise"]]
try:
with transaction.atomic():
proxy.save()
except IntegrityError as e:
raise e
class Command(BaseCommand):
"""
Command for importing the database of NK15.
Need to be run by a user with a registered role in postgres for the database nk15.
"""
def print_success(self, to_print):
return self.stdout.write(self.style.SUCCESS(to_print))
def add_arguments(self, parser):
parser.add_argument('-c', '--comptes', action='store_true', help="import accounts")
parser.add_argument('-b', '--boutons', action='store_true', help="import boutons")
parser.add_argument('-t', '--transactions', action='store_true', help="import transaction")
parser.add_argument('-al', '--aliases', action='store_true', help="import aliases")
parser.add_argument('-ac', '--activities', action='store_true', help="import activities")
parser.add_argument('-r', '--remittances', action='store_true', help="import check remittances")
parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name')
parser.add_argument('-u', '--nk15user', action='store', default='nk15_user', help='NK15 database owner')
def handle(self, *args, **kwargs):
global MAP_IDBDE
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
# connecting to nk15 database
conn = pg.connect(database=nk15db, user=nk15user)
cur = conn.cursor(cursor_factory=pge.DictCursor)
if kwargs["comptes"]:
# reset database.
call_command("migrate")
call_command("loaddata", "initial")
self.print_success("reset nk20 database\n")
import_comptes(cur)
self.print_success("comptes table imported")
elif kwargs["map"]:
filename = kwargs["map"]
with open(filename, 'r') as fp:
MAP_IDBDE = json.load(fp)
MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()}
if kwargs["save"]:
filename = kwargs["save"]
with open(filename, 'w') as fp:
json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
# /!\ need a prober MAP_IDBDE
if kwargs["boutons"]:
import_boutons(cur)
self.print_success("boutons table imported\n")
if kwargs["activities"]:
import_activities(cur)
self.print_success("activities imported\n")
import_activity_entries(cur)
self.print_success("activity entries imported\n")
if kwargs["aliases"]:
import_aliases(cur)
self.print_success("aliases imported\n")
if kwargs["transactions"]:
import_transaction(cur)
self.print_success("transaction imported\n")
if kwargs["remittances"]:
import_remittances(cur)
self.print_success("remittances imported\n")