ynerant
/
nk20-scripts
mirror of https://gitlab.crans.org/bde/nk20-scripts
1
0
Fork 0
Code Issues Releases Wiki Activity

Better import

This commit is contained in:
Yohann D'ANELLO 2020-07-22 01:43:18 +02:00
parent 4839b2deb8 dc1daf0a2d
commit 6cfae5fd69
5 changed files with 102 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
management/commands/_import_utils.py
View File

@ -44,7 +44,7 @@ class ImportCommand(BaseCommand):
n = str(n)
total = str(total)
n.rjust(len(total))
print(f"\r ({n}/{total}) {content:10.10}", end="")
print(f"\r ({n}/{total}) {content:16.16}", end="")
def create_parser(self, prog_name, subcommand, **kwargs):
parser = super().create_parser(prog_name, subcommand, **kwargs)

4
management/commands/import_nk15.py
View File

@ -30,4 +30,6 @@ class Command(ImportCommand):
kwargs["map"] = "map.json"
kwargs["buttons"] = True
call_command('import_transaction', **kwargs)
#
call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant')
call_command('syncsql')

84
management/commands/import_transaction.py
View File

@ -8,12 +8,14 @@ import copy
from django.utils.timezone import make_aware
from django.db import transaction
from django.contrib.contenttypes.models import ContentType
from note.models import (TemplateCategory,
TransactionTemplate,
Transaction,
RecurrentTransaction,
SpecialTransaction
SpecialTransaction,
MembershipTransaction,
)
from note.models import Note, NoteClub
from activity.models import Guest, GuestTransaction
@ -21,15 +23,27 @@ from activity.models import Guest, GuestTransaction
from member.models import Membership, MembershipTransaction, Role
from ._import_utils import ImportCommand, BulkCreateManager, timed
# from member/fixtures/initial
BDE_PK = 1
KFET_PK = 2
# from note/fixtures/initial
NOTE_SPECIAL_CODE = {
"espèce": 1,
"carte": 2,
"chèque": 3,
"virement": 4,
}
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2
CT = {
"RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
"SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
"MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
"GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}
def get_date_end(date_start):
date_end = copy.deepcopy(date_start)
@ -79,22 +93,24 @@ class Command(ImportCommand):
def _basic_transaction(self, row, obj_dict, child_dict):
if len(row["description"]) > 255:
obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
return obj_dict, None, None
def _template_transaction(self, row, obj_dict, child_dict):
if self.categories.get(row["categorie"]):
child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
elif self.buttons.get(row["description"]):
if self.buttons.get(row["description"]):
child_dict["category_id"] = self.buttons[row["description"]][1]
child_dict["template_id"] = self.buttons[row["description"]][0]
# elif self.categories.get(row["categorie"]):
# child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
else:
return obj_dict, None, None
obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
return obj_dict, child_dict, RecurrentTransaction
def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
obj_dict2 = obj_dict.copy()
child_dict2 = child_dict.copy()
child_dict2["membership_id"] = pk_membership
@ -104,6 +120,7 @@ class Command(ImportCommand):
def _special_transaction(self, row, obj_dict, child_dict):
# Some transaction uses BDE (idbde=0) as source or destination,
# lets fix that.
obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
field_id = "source_id" if row["type"] == "crédit" else "destination_id"
if "espèce" in row["description"]:
obj_dict[field_id] = 1
@ -128,13 +145,14 @@ class Command(ImportCommand):
def _guest_transaction(self, row, obj_dict, child_dict):
# Currently GuestTransaction is related to a Guest.
# This is not ideal and should be change to the Entry of this Guest.
obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
if m:
first_name, last_name = m.group(2), m.group(3)
if first_name == "Marion" and last_name == "Bizu Pose":
first_name, last_name = "Marion Bizu", "Pose"
guest_id = Guest.objects.filter(first_name__iexact=first_name,
last_name__iexact=last_name).first().pk
last_name__iexact=last_name).first().pk
child_dict["guest_id"] = guest_id
else:
raise(f"Guest not Found {row['id']} {first_name}, last_name")
@ -171,35 +189,42 @@ class Command(ImportCommand):
obj_dict = {
"pk": pk_transaction,
"destination_id": self.MAP_IDBDE[row["destinataire"]],
"polymorphic_ctype": None,
"source_id": self.MAP_IDBDE[row["emetteur"]],
"created_at": date,
"amount": row["montant"],
"created_at": date,
"destination_alias": "",
"invalidity_reason": None,
"quantity": row["quantite"],
"reason": row["description"],
"source_alias": "",
"valid": row["valide"],
}
# for child transaction Models
child_dict = {"pk": obj_dict["pk"]}
child_dict = {"pk": pk_transaction}
ttype = row["type"]
# Membership transaction detection and import
if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=obj_dict["source_id"])
if isinstance(note, NoteClub):
child_transaction = None
child_transaction = None # don't bother register clubs
else:
user_id = note.user_id
montant = obj_dict["amount"]
obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
(obj_dict0,
child_dict0,
child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
bde_dict = {
"pk": pk_membership,
"user_id": user_id,
"club_id": KFET_PK,
"club_id": BDE_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": min(500, montant)
}
pk_membership += 1
pk_transaction += 1
obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
# Kfet membership
# BDE Membership
obj_dict["pk"] = pk_transaction
@ -207,7 +232,7 @@ class Command(ImportCommand):
kfet_dict = {
"pk": pk_membership,
"user_id": user_id,
"club_id": BDE_PK,
"club_id": KFET_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": max(montant - 500, 0),
@ -218,12 +243,12 @@ class Command(ImportCommand):
pk_membership += 1
pk_transaction += 1
bulk_mgr.add(
Membership(**bde_dict),
Membership(**kfet_dict),
Transaction(**obj_dict0),
child_transaction(**child_dict0),
Transaction(**obj_dict),
child_transaction(**child_dict),
Membership(**bde_dict),
Membership(**kfet_dict),
)
continue
elif ttype == "bouton":
@ -232,7 +257,7 @@ class Command(ImportCommand):
obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
elif ttype == "invitation":
obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
if ttype == "don" or ttype == "transfert":
elif ttype == "don" or ttype == "transfert":
obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
else:
child_transaction = None
@ -243,18 +268,15 @@ class Command(ImportCommand):
pk_transaction += 1
bulk_mgr.done()
@timed
def adjust_roles(self):
bdeRole = Role.objects.get(name="Adhérent BDE")
kfetRole = Role.objects.get(name="Adhérent Kfet")
memberships = Membership.objects.all()
n = len(memberships)
for idx, membership in enumerate(memberships):
self.update_line(idx, n, membership.user.username)
if membership.club.name == "BDE":
membership.roles.set([bdeRole])
elif membership.club.name == "Kfet":
membership.roles.set([kfetRole])
def set_roles(self):
bulk_mgr = BulkCreateManager(chunk_size=10000)
membership_ids = Membership.objects.values_list('id',flat=True)
for m_id in membership_ids:
bulk_mgr.add(
Membership.roles.through(membership_id=m_id,role_id=BDE_ROLE_PK),
Membership.roles.through(membership_id=m_id,role_id=KFET_ROLE_PK),
)
bulk_mgr.done()
@timed
def handle(self, *args, **kwargs):
@ -268,4 +290,4 @@ class Command(ImportCommand):
self.load_map(kwargs["map"])
self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"])
self.adjust_roles()
self.set_roles()

43
management/commands/syncsql.py Normal file
View File

@ -0,0 +1,43 @@
#!/usr/bin/env python3
from django.core.management.base import BaseCommand
from django.apps import apps
from django.db import connection
from polymorphic.models import PolymorphicModel
NO_SEQ = [
"Session",
"Token",
"WEIRole", # dirty fix
]
class Command(BaseCommand):
"""
Command to synchronise primary sequence of postgres after bulk insert of django.
"""
def add_arguments(self,parser):
parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
return parser
def handle(self, *args, **kwargs):
app_list = kwargs["apps"]
if len(app_list):
model_classes = list()
for app in app_list:
model_classes += apps.get_app_config(app).get_models()
else:
# no app specified, sync everything
model_classes = apps.get_models(include_auto_created=True)
db_names = [ m._meta.db_table for m in model_classes if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count()>1]
com = "BEGIN;\n"
for db_name in db_names:
com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "{db_name}";\n'
com += "COMMIT;"
print(com)
cur = connection.cursor()
cur.execute(com)
cur.close()

2
shell/tabularasa
View File

@ -1,4 +1,6 @@
#!/usr/bin/sh
sudo service postgresql stop
sudo service postgresql start
sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
echo 'reset db';
find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete