ynerant
/
nk20-scripts
mirror of https://gitlab.crans.org/bde/nk20-scripts
1
0
Fork 0
Code Issues Releases Wiki Activity

Import activities, guests and guest transactions

This commit is contained in:
Yohann D'ANELLO 2020-04-27 01:19:56 +02:00
parent 1090fe0b26
commit 4ce9aa0d4a
1 changed files with 213 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
management/commands/import_nk15.py
View File

@ -2,25 +2,24 @@
from django.core.management.base import BaseCommand from django.core.management.base import BaseCommand
from django.core.management import call_command from django.core.management import call_command
from django.utils import timezone
import psycopg2 as pg import psycopg2 as pg
import psycopg2.extras as pge import psycopg2.extras as pge
from django.db import transaction from django.db import transaction
import json import json
import datetime import datetime
import collections import re
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.utils.timezone import make_aware from django.utils.timezone import make_aware
from django.db import IntegrityError from django.db import IntegrityError
from django.contrib.auth.models import User from django.contrib.auth.models import User
from note.models import Note, NoteSpecial, NoteUser, NoteClub from activity.models import ActivityType, Activity, Guest, Entry, GuestTransaction
from note.models import Note
from note.models import Alias from note.models import Alias
from note.models import TemplateCategory, TransactionTemplate,\ from note.models import TemplateCategory, TransactionTemplate, \
Transaction, RecurrentTransaction, MembershipTransaction, SpecialTransaction Transaction, RecurrentTransaction, SpecialTransaction
from member.models import Profile, Club, Membership from member.models import Club
""" """
Script d'import de la nk15: Script d'import de la nk15:
@ -31,11 +30,10 @@ TODO: import ...
""" """
M_DURATION = 396 M_DURATION = 396
M_START = datetime.date(2019,8,31) M_START = datetime.date(2019, 8, 31)
M_END = datetime.date(2020,9,30) M_END = datetime.date(2020, 9, 30)
MAP_IDBDE = {
MAP_IDBDE={
-4: 2, # Carte Bancaire -4: 2, # Carte Bancaire
-3: 4, # Virement -3: 4, # Virement
-2: 1, # Especes -2: 1, # Especes
@ -43,23 +41,29 @@ MAP_IDBDE={
0: 5, # BDE 0: 5, # BDE
} }
def update_line(n,N, content): MAP_IDACTIVITY = {}
MAP_NAMEACTIVITY = {}
MAP_NAMEGUEST = {}
def update_line(n, total, content):
n = str(n) n = str(n)
N = str(N) total = str(total)
n.rjust(len(N)) n.rjust(len(total))
print(f"\r ({n}/{N}) {content:10.10}",end="") print(f"\r ({n}/{total}) {content:10.10}", end="")
@transaction.atomic @transaction.atomic
def import_comptes(cur): def import_comptes(cur):
cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;") cur.execute("SELECT * FROM comptes WHERE idbde > 0 ORDER BY idbde;")
pkclub = 3 pkclub = 3
N = cur.rowcount n = cur.rowcount
for idx, row in enumerate(cur): for idx, row in enumerate(cur):
update_line(idx,N,row["pseudo"]) update_line(idx, n, row["pseudo"])
if row["type"] == "personne": if row["type"] == "personne":
#sanitize password # sanitize password
if row["passwd"] != "*|*": if row["passwd"] != "*|*":
passwd_nk15 = "$".join(["custom_nk15","1",row["passwd"]]) passwd_nk15 = "$".join(["custom_nk15", "1", row["passwd"]])
else: else:
passwd_nk15 = '' passwd_nk15 = ''
try: try:
@ -69,7 +73,7 @@ def import_comptes(cur):
"first_name": row["nom"], "first_name": row["nom"],
"last_name": row["prenom"], "last_name": row["prenom"],
"email": row["mail"], "email": row["mail"],
"is_active" : True, # temporary "is_active": True, # temporary
} }
user = User.objects.create(**obj_dict) user = User.objects.create(**obj_dict)
@ -84,32 +88,31 @@ def import_comptes(cur):
# sanitize duplicate aliases (nk12) # sanitize duplicate aliases (nk12)
except ValidationError as e: except ValidationError as e:
if e.code == 'same_alias': if e.code == 'same_alias':
user.username = row["pseudo"]+str(row["idbde"]) user.username = row["pseudo"] + str(row["idbde"])
user.save() user.save()
else: else:
raise(e) raise e
# profile and note created via signal. # profile and note created via signal.
note = user.note note = user.note
date = row.get("last_negatif",None) date = row.get("last_negatif", None)
if date != None: if date is not None:
note.last_negative = make_aware(date) note.last_negative = make_aware(date)
note.balance = row["solde"] note.balance = row["solde"]
obj_list =[user, profile, note]
note.save() note.save()
else: # club else: # club
obj_dict = { obj_dict = {
"pk":pkclub, "pk": pkclub,
"name": row["pseudo"], "name": row["pseudo"],
"email": row["mail"], "email": row["mail"],
"membership_duration": M_DURATION, "membership_duration": M_DURATION,
"membership_start": M_START, "membership_start": M_START,
"membership_end": M_END, "membership_end": M_END,
"membership_fee_paid": 0, "membership_fee_paid": 0,
"membership_fee_unpaid":0, "membership_fee_unpaid": 0,
} }
club,c = Club.objects.get_or_create(**obj_dict) club, c = Club.objects.get_or_create(**obj_dict)
pkclub +=1 pkclub += 1
note = club.note note = club.note
note.balance = row["solde"] note.balance = row["solde"]
club.save() club.save()
@ -117,12 +120,13 @@ def import_comptes(cur):
MAP_IDBDE[row["idbde"]] = note.note_ptr_id MAP_IDBDE[row["idbde"]] = note.note_ptr_id
@transaction.atomic @transaction.atomic
def import_boutons(cur): def import_boutons(cur):
cur.execute("SELECT * FROM boutons;") cur.execute("SELECT * FROM boutons;")
N = cur.rowcount n = cur.rowcount
for idx, row in enumerate(cur): for idx, row in enumerate(cur):
update_line(idx,N,row["label"]) update_line(idx, n, row["label"])
cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"]) cat, created = TemplateCategory.objects.get_or_create(name=row["categorie"])
if created: if created:
cat.save() cat.save()
@ -132,7 +136,7 @@ def import_boutons(cur):
"amount": row["montant"], "amount": row["montant"],
"destination_id": MAP_IDBDE[row["destinataire"]], "destination_id": MAP_IDBDE[row["destinataire"]],
"category": cat, "category": cat,
"display" : row["affiche"], "display": row["affiche"],
"description": row["description"], "description": row["description"],
} }
try: try:
@ -143,49 +147,49 @@ def import_boutons(cur):
if "unique" in e.args[0]: if "unique" in e.args[0]:
qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name') qs = Club.objects.filter(note__note_ptr=MAP_IDBDE[row["destinataire"]]).values('name')
note_name = qs[0]["name"] note_name = qs[0]["name"]
#rename button name # rename button name
obj_dict["name"] ="{} {}".format(obj_dict["name"],note_name) obj_dict["name"] = "{} {}".format(obj_dict["name"], note_name)
button = TransactionTemplate.objects.create(**obj_dict) button = TransactionTemplate.objects.create(**obj_dict)
else: else:
raise(e) raise e
button.save() button.save()
@transaction.atomic @transaction.atomic
def import_transaction(cur): def import_transaction(cur):
idmin=58770 idmin = 58770
cur.execute("SELECT *, transactions.date AS transac_date\ cur.execute("SELECT *, transactions.date AS transac_date\
FROM transactions\ FROM transactions\
LEFT JOIN adhesions ON transactions.id = adhesions.id\ LEFT JOIN adhesions ON transactions.id = adhesions.id\
WHERE transactions.id> {}\ WHERE transactions.id> {}\
ORDER BY transactions.id;".format(idmin)) ORDER BY transactions.id;".format(idmin))
N = cur.rowcount n = cur.rowcount
transac_list = []
for idx, row in enumerate(cur): for idx, row in enumerate(cur):
update_line(idx,N,row["description"]) update_line(idx, n, row["description"])
# some date are set to None, use the previous one # some date are set to None, use the previous one
date = row["transac_date"] date = row["transac_date"]
obj_dict = { obj_dict = {
# "pk": row["id"], # "pk": row["id"],
"destination_id" : MAP_IDBDE[row["destinataire"]], "destination_id": MAP_IDBDE[row["destinataire"]],
"source_id": MAP_IDBDE[row["emetteur"]], "source_id": MAP_IDBDE[row["emetteur"]],
"created_at":make_aware(date), "created_at": make_aware(date),
"amount":row["montant"], "amount": row["montant"],
"quantity":row["quantite"], "quantity": row["quantite"],
"reason":row["description"], "reason": row["description"],
"valid":row["valide"], "valid": row["valide"],
} }
ttype = row["type"] ttype = row["type"]
if ttype == "don" or ttype == "transfert" or ttype == "invitation": if ttype == "don" or ttype == "transfert":
transac = Transaction.objects.create(**obj_dict) Transaction.objects.create(**obj_dict)
elif ttype == "bouton": elif ttype == "bouton":
cat_name = row["categorie"] cat_name = row["categorie"]
if cat_name == None: if cat_name is None:
cat_name = 'None' cat_name = 'None'
cat, created = TemplateCategory.objects.get_or_create(name=cat_name) cat, created = TemplateCategory.objects.get_or_create(name=cat_name)
if created: if created:
cat.save() cat.save()
obj_dict["category"] = cat obj_dict["category"] = cat
transac = RecurrentTransaction.objects.create(**obj_dict) RecurrentTransaction.objects.create(**obj_dict)
elif ttype == "crédit" or ttype == "retrait": elif ttype == "crédit" or ttype == "retrait":
field_id = "source_id" if ttype == "crédit" else "destination_id" field_id = "source_id" if ttype == "crédit" else "destination_id"
if "espèce" in row["description"]: if "espèce" in row["description"]:
@ -206,25 +210,53 @@ def import_transaction(cur):
obj_dict["first_name"] = actor.club.name obj_dict["first_name"] = actor.club.name
obj_dict["last_name"] = actor.club.name obj_dict["last_name"] = actor.club.name
else: else:
raise("You should'nt be there") raise Exception("You should'nt be there")
transac = SpecialTransaction.objects.create(**obj_dict) SpecialTransaction.objects.create(**obj_dict)
elif ttype == "adhésion": elif ttype == "adhésion":
print("adhesion not supported yet") print("adhesion not supported yet")
elif ttype == "invitation":
m = re.search("Invitation (.*?) \((.*?)\)", row["description"])
if m is None:
raise IntegrityError("Invitation is not well formated: {} (must be 'Invitation ACTIVITY_NAME (NAME)')"
.format(row["description"]))
activity_name = m.group(1)
guest_name = m.group(2)
if activity_name not in MAP_NAMEACTIVITY:
raise IntegrityError("Activity {} is not found".format(activity_name,))
activity = MAP_NAMEACTIVITY[activity_name]
if guest_name not in MAP_NAMEGUEST:
raise IntegrityError("Guest {} is not found".format(guest_name,))
guest = None
for g in MAP_NAMEGUEST[guest_name]:
if g.activity.pk == activity.pk:
guest = g
break
if guest is None:
raise IntegrityError("Guest {} didn't go to the activity {}".format(guest_name, activity_name,))
obj_dict["guest"] = guest
GuestTransaction.objects.get_or_create(**obj_dict)
else: else:
print("other type not supported yet") print("other type not supported yet:", ttype)
@transaction.atomic @transaction.atomic
def import_aliases(cur): def import_aliases(cur):
cur.execute("SELECT * FROM aliases ORDER by id") cur.execute("SELECT * FROM aliases ORDER by id")
N = cur.rowcount n = cur.rowcount
for idx, row in enumerate(cur): for idx, row in enumerate(cur):
update_line(idx,N,row["alias"]) update_line(idx, n, row["titre"])
alias_name = row["alias"] alias_name = row["alias"]
alias_name_good = (alias_name[:252]+'...') if len(alias_name) > 255 else alias_name alias_name_good = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
obj_dict = { obj_dict = {
"note_id":MAP_IDBDE[row["idbde"]], "note_id": MAP_IDBDE[row["idbde"]],
"name":alias_name_good, "name": alias_name_good,
"normalized_name":Alias.normalize(alias_name_good) "normalized_name": Alias.normalize(alias_name_good)
} }
try: try:
with transaction.atomic(): with transaction.atomic():
@ -234,23 +266,112 @@ def import_aliases(cur):
if "unique" in e.args[0]: if "unique" in e.args[0]:
continue continue
else: else:
raise(e) raise e
alias.save() alias.save()
@transaction.atomic
def import_activities(cur):
cur.execute("SELECT * FROM activites ORDER by id")
n = cur.rowcount
activity_type = ActivityType.objects.get(name="Pot") # Need to be fixed manually
kfet = Club.objects.get(name="Kfet")
for idx, row in enumerate(cur):
update_line(idx, n, row["alias"])
organizer = Club.objects.filter(name=row["signature"])
if organizer.exists():
# Try to find the club that organizes the activity. If not founded, assume that is Kfet (fix manually)
organizer = organizer.get()
else:
organizer = kfet
obj_dict = {
"name": row["titre"],
"description": row["description"],
"activity_type": activity_type, # By default Pot
"creater": MAP_IDBDE[row["responsable"]],
"organizer": organizer,
"attendees_club": kfet, # Maybe fix manually
"date_start": row["debut"],
"date_end": row["fin"],
"valid": row["validepar"] is not None,
"open": row["open"], # Should be always False
}
# WARNING: Fields lieu, liste, listeimprimee are missing
try:
with transaction.atomic():
activity = Activity.objects.get_or_create(**obj_dict)[0]
MAP_IDACTIVITY[row["id"]] = activity
MAP_NAMEACTIVITY[activity.name] = activity
except IntegrityError as e:
raise e
@transaction.atomic
def import_activity_entries(cur):
map_idguests = {}
cur.execute("SELECT * FROM invites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
obj_dict = {
"activity": MAP_IDACTIVITY[row["activity"]],
"last_name": row["nom"],
"first_name": row["prenom"],
"inviter": MAP_IDBDE[row["responsable"]],
}
try:
with transaction.atomic():
guest = Guest.objects.get_or_create(**obj_dict)[0]
map_idguests.setdefault(row["responsable"], [])
map_idguests[row["id"]].append(guest)
guest_name = guest.first_name + " " + guest.last_name
MAP_NAMEGUEST.setdefault(guest_name, [])
MAP_NAMEGUEST[guest_name].append(guest)
except IntegrityError as e:
raise e
cur.execute("SELECT * FROM entree_activites ORDER by id")
n = cur.rowcount
for idx, row in enumerate(cur):
update_line(idx, n, row["nom"] + " " + row["prenom"])
activity = MAP_IDACTIVITY[row["activity"]]
guest = None
if row["est_invite"]:
for g in map_idguests[row["id"]]:
if g.activity.pk == activity.pk:
guest = g
break
if not guest:
raise IntegrityError("Guest was not found: " + str(row))
obj_dict = {
"activity": activity,
"time": row["heure_entree"],
"note": guest.inviter if guest else MAP_IDBDE[row["idbde"]],
"guest": guest,
}
try:
with transaction.atomic():
Entry.objects.get_or_create(**obj_dict)
except IntegrityError as e:
raise e
class Command(BaseCommand): class Command(BaseCommand):
""" """
Command for importing the database of NK15. Command for importing the database of NK15.
Need to be run by a user with a registered role in postgres for the database nk15. Need to be run by a user with a registered role in postgres for the database nk15.
""" """
def print_success(self,to_print):
def print_success(self, to_print):
return self.stdout.write(self.style.SUCCESS(to_print)) return self.stdout.write(self.style.SUCCESS(to_print))
def add_arguments(self,parser): def add_arguments(self, parser):
parser.add_argument('-c', '--comptes', action = 'store_true', help="import accounts") parser.add_argument('-c', '--comptes', action='store_true', help="import accounts")
parser.add_argument('-b', '--boutons', action = 'store_true', help="import boutons") parser.add_argument('-b', '--boutons', action='store_true', help="import boutons")
parser.add_argument('-t', '--transactions', action = 'store_true',help="import transaction") parser.add_argument('-t', '--transactions', action='store_true', help="import transaction")
parser.add_argument('-a', '--aliases', action = 'store_true',help="import aliases") parser.add_argument('-al', '--aliases', action='store_true', help="import aliases")
parser.add_argument('-ac', '--activities', action='store_true', help="import activities")
parser.add_argument('-s', '--save', action='store', help="save mapping of idbde") parser.add_argument('-s', '--save', action='store', help="save mapping of idbde")
parser.add_argument('-m', '--map', action='store', help="import mapping of idbde") parser.add_argument('-m', '--map', action='store', help="import mapping of idbde")
parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name') parser.add_argument('-d', '--nk15db', action='store', default='nk15', help='NK15 database name')
@ -260,30 +381,33 @@ class Command(BaseCommand):
global MAP_IDBDE global MAP_IDBDE
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user'] nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
# connecting to nk15 database # connecting to nk15 database
conn = pg.connect(database=nk15db,user=nk15user) conn = pg.connect(database=nk15db, user=nk15user)
cur = conn.cursor(cursor_factory = pge.DictCursor) cur = conn.cursor(cursor_factory=pge.DictCursor)
if kwargs["comptes"]: if kwargs["comptes"]:
#reset database. # reset database.
call_command("migrate") call_command("migrate")
call_command("loaddata","initial") call_command("loaddata", "initial")
self.print_success("reset nk20 database\n") self.print_success("reset nk20 database\n")
import_comptes(cur) import_comptes(cur)
self.print_success("comptes table imported") self.print_success("comptes table imported")
elif kwargs["map"]: elif kwargs["map"]:
filename = kwargs["map"] filename = kwargs["map"]
with open(filename,'r') as fp: with open(filename, 'r') as fp:
MAP_IDBDE = json.load(fp) MAP_IDBDE = json.load(fp)
MAP_IDBDE = {int(k):int(v) for k,v in MAP_IDBDE.items()} MAP_IDBDE = {int(k): int(v) for k, v in MAP_IDBDE.items()}
if kwargs["save"]: if kwargs["save"]:
filename = kwargs["save"] filename = kwargs["save"]
with open(filename,'w') as fp: with open(filename, 'w') as fp:
json.dump(MAP_IDBDE,fp,sort_keys=True, indent=2) json.dump(MAP_IDBDE, fp, sort_keys=True, indent=2)
# /!\ need a prober MAP_IDBDE # /!\ need a prober MAP_IDBDE
if kwargs["boutons"]: if kwargs["boutons"]:
import_boutons(cur) import_boutons(cur)
self.print_success("boutons table imported\n") self.print_success("boutons table imported\n")
if kwargs["activities"]:
import_activities(cur)
self.print_success("activities imported\n")
if kwargs["aliases"]: if kwargs["aliases"]:
import_aliases(cur) import_aliases(cur)
self.print_success("aliases imported\n") self.print_success("aliases imported\n")