ynerant
/
nk20-scripts
mirror of https://gitlab.crans.org/bde/nk20-scripts
1
0
Fork 0
Code Issues Releases Wiki Activity

Better import

This commit is contained in:
Yohann D'ANELLO 2020-07-22 01:43:18 +02:00
parent 4839b2deb8 dc1daf0a2d
commit 6cfae5fd69
5 changed files with 102 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
management/commands/_import_utils.py
View File

@ -44,7 +44,7 @@ class ImportCommand(BaseCommand):
n = str(n) n = str(n)
total = str(total) total = str(total)
n.rjust(len(total)) n.rjust(len(total))
print(f"\r ({n}/{total}) {content:10.10}", end="") print(f"\r ({n}/{total}) {content:16.16}", end="")
def create_parser(self, prog_name, subcommand, **kwargs): def create_parser(self, prog_name, subcommand, **kwargs):
parser = super().create_parser(prog_name, subcommand, **kwargs) parser = super().create_parser(prog_name, subcommand, **kwargs)

4
management/commands/import_nk15.py
View File

@ -30,4 +30,6 @@ class Command(ImportCommand):
kwargs["map"] = "map.json" kwargs["map"] = "map.json"
kwargs["buttons"] = True kwargs["buttons"] = True
call_command('import_transaction', **kwargs) call_command('import_transaction', **kwargs)
#
call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant')
call_command('syncsql')

84
management/commands/import_transaction.py
View File

@ -8,12 +8,14 @@ import copy
from django.utils.timezone import make_aware from django.utils.timezone import make_aware
from django.db import transaction from django.db import transaction
from django.contrib.contenttypes.models import ContentType
from note.models import (TemplateCategory, from note.models import (TemplateCategory,
TransactionTemplate, TransactionTemplate,
Transaction, Transaction,
RecurrentTransaction, RecurrentTransaction,
SpecialTransaction SpecialTransaction,
MembershipTransaction,
) )
from note.models import Note, NoteClub from note.models import Note, NoteClub
from activity.models import Guest, GuestTransaction from activity.models import Guest, GuestTransaction
@ -21,15 +23,27 @@ from activity.models import Guest, GuestTransaction
from member.models import Membership, MembershipTransaction, Role from member.models import Membership, MembershipTransaction, Role
from ._import_utils import ImportCommand, BulkCreateManager, timed from ._import_utils import ImportCommand, BulkCreateManager, timed
# from member/fixtures/initial
BDE_PK = 1 BDE_PK = 1
KFET_PK = 2 KFET_PK = 2
# from note/fixtures/initial
NOTE_SPECIAL_CODE = { NOTE_SPECIAL_CODE = {
"espèce": 1, "espèce": 1,
"carte": 2, "carte": 2,
"chèque": 3, "chèque": 3,
"virement": 4, "virement": 4,
} }
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2
CT = {
"RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
"SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
"MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
"GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}
def get_date_end(date_start): def get_date_end(date_start):
date_end = copy.deepcopy(date_start) date_end = copy.deepcopy(date_start)
@ -79,22 +93,24 @@ class Command(ImportCommand):
def _basic_transaction(self, row, obj_dict, child_dict): def _basic_transaction(self, row, obj_dict, child_dict):
if len(row["description"]) > 255: if len(row["description"]) > 255:
obj_dict["reason"] = obj_dict["reason"][:250]+"...)" obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
return obj_dict, None, None return obj_dict, None, None
def _template_transaction(self, row, obj_dict, child_dict): def _template_transaction(self, row, obj_dict, child_dict):
if self.categories.get(row["categorie"]): if self.buttons.get(row["description"]):
child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
elif self.buttons.get(row["description"]):
child_dict["category_id"] = self.buttons[row["description"]][1] child_dict["category_id"] = self.buttons[row["description"]][1]
child_dict["template_id"] = self.buttons[row["description"]][0] child_dict["template_id"] = self.buttons[row["description"]][0]
# elif self.categories.get(row["categorie"]):
# child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
else: else:
return obj_dict, None, None return obj_dict, None, None
obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
return obj_dict, child_dict, RecurrentTransaction return obj_dict, child_dict, RecurrentTransaction
def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
obj_dict2 = obj_dict.copy() obj_dict2 = obj_dict.copy()
child_dict2 = child_dict.copy() child_dict2 = child_dict.copy()
child_dict2["membership_id"] = pk_membership child_dict2["membership_id"] = pk_membership
@ -104,6 +120,7 @@ class Command(ImportCommand):
def _special_transaction(self, row, obj_dict, child_dict): def _special_transaction(self, row, obj_dict, child_dict):
# Some transaction uses BDE (idbde=0) as source or destination, # Some transaction uses BDE (idbde=0) as source or destination,
# lets fix that. # lets fix that.
obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
field_id = "source_id" if row["type"] == "crédit" else "destination_id" field_id = "source_id" if row["type"] == "crédit" else "destination_id"
if "espèce" in row["description"]: if "espèce" in row["description"]:
obj_dict[field_id] = 1 obj_dict[field_id] = 1
@ -128,13 +145,14 @@ class Command(ImportCommand):
def _guest_transaction(self, row, obj_dict, child_dict): def _guest_transaction(self, row, obj_dict, child_dict):
# Currently GuestTransaction is related to a Guest. # Currently GuestTransaction is related to a Guest.
# This is not ideal and should be change to the Entry of this Guest. # This is not ideal and should be change to the Entry of this Guest.
obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
if m: if m:
first_name, last_name = m.group(2), m.group(3) first_name, last_name = m.group(2), m.group(3)
if first_name == "Marion" and last_name == "Bizu Pose": if first_name == "Marion" and last_name == "Bizu Pose":
first_name, last_name = "Marion Bizu", "Pose" first_name, last_name = "Marion Bizu", "Pose"
guest_id = Guest.objects.filter(first_name__iexact=first_name, guest_id = Guest.objects.filter(first_name__iexact=first_name,
last_name__iexact=last_name).first().pk last_name__iexact=last_name).first().pk
child_dict["guest_id"] = guest_id child_dict["guest_id"] = guest_id
else: else:
raise(f"Guest not Found {row['id']} {first_name}, last_name") raise(f"Guest not Found {row['id']} {first_name}, last_name")
@ -171,35 +189,42 @@ class Command(ImportCommand):
obj_dict = { obj_dict = {
"pk": pk_transaction, "pk": pk_transaction,
"destination_id": self.MAP_IDBDE[row["destinataire"]], "destination_id": self.MAP_IDBDE[row["destinataire"]],
"polymorphic_ctype": None,
"source_id": self.MAP_IDBDE[row["emetteur"]], "source_id": self.MAP_IDBDE[row["emetteur"]],
"created_at": date,
"amount": row["montant"], "amount": row["montant"],
"created_at": date,
"destination_alias": "",
"invalidity_reason": None,
"quantity": row["quantite"], "quantity": row["quantite"],
"reason": row["description"], "reason": row["description"],
"source_alias": "",
"valid": row["valide"], "valid": row["valide"],
} }
# for child transaction Models # for child transaction Models
child_dict = {"pk": obj_dict["pk"]} child_dict = {"pk": pk_transaction}
ttype = row["type"] ttype = row["type"]
# Membership transaction detection and import
if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=obj_dict["source_id"]) note = Note.objects.get(pk=obj_dict["source_id"])
if isinstance(note, NoteClub): if isinstance(note, NoteClub):
child_transaction = None child_transaction = None # don't bother register clubs
else: else:
user_id = note.user_id user_id = note.user_id
montant = obj_dict["amount"] montant = obj_dict["amount"]
obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) (obj_dict0,
child_dict0,
child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
bde_dict = { bde_dict = {
"pk": pk_membership, "pk": pk_membership,
"user_id": user_id, "user_id": user_id,
"club_id": KFET_PK, "club_id": BDE_PK,
"date_start": date.date(), # Only date, not time "date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()), "date_end": get_date_end(date.date()),
"fee": min(500, montant) "fee": min(500, montant)
} }
pk_membership += 1 pk_membership += 1
pk_transaction += 1 pk_transaction += 1
obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
# Kfet membership # Kfet membership
# BDE Membership # BDE Membership
obj_dict["pk"] = pk_transaction obj_dict["pk"] = pk_transaction
@ -207,7 +232,7 @@ class Command(ImportCommand):
kfet_dict = { kfet_dict = {
"pk": pk_membership, "pk": pk_membership,
"user_id": user_id, "user_id": user_id,
"club_id": BDE_PK, "club_id": KFET_PK,
"date_start": date.date(), # Only date, not time "date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()), "date_end": get_date_end(date.date()),
"fee": max(montant - 500, 0), "fee": max(montant - 500, 0),
@ -218,12 +243,12 @@ class Command(ImportCommand):
pk_membership += 1 pk_membership += 1
pk_transaction += 1 pk_transaction += 1
bulk_mgr.add( bulk_mgr.add(
Membership(**bde_dict),
Membership(**kfet_dict),
Transaction(**obj_dict0), Transaction(**obj_dict0),
child_transaction(**child_dict0), child_transaction(**child_dict0),
Transaction(**obj_dict), Transaction(**obj_dict),
child_transaction(**child_dict), child_transaction(**child_dict),
Membership(**bde_dict),
Membership(**kfet_dict),
) )
continue continue
elif ttype == "bouton": elif ttype == "bouton":
@ -232,7 +257,7 @@ class Command(ImportCommand):
obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
elif ttype == "invitation": elif ttype == "invitation":
obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
if ttype == "don" or ttype == "transfert": elif ttype == "don" or ttype == "transfert":
obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
else: else:
child_transaction = None child_transaction = None
@ -243,18 +268,15 @@ class Command(ImportCommand):
pk_transaction += 1 pk_transaction += 1
bulk_mgr.done() bulk_mgr.done()
@timed def set_roles(self):
def adjust_roles(self): bulk_mgr = BulkCreateManager(chunk_size=10000)
bdeRole = Role.objects.get(name="Adhérent BDE") membership_ids = Membership.objects.values_list('id',flat=True)
kfetRole = Role.objects.get(name="Adhérent Kfet") for m_id in membership_ids:
memberships = Membership.objects.all() bulk_mgr.add(
n = len(memberships) Membership.roles.through(membership_id=m_id,role_id=BDE_ROLE_PK),
for idx, membership in enumerate(memberships): Membership.roles.through(membership_id=m_id,role_id=KFET_ROLE_PK),
self.update_line(idx, n, membership.user.username) )
if membership.club.name == "BDE": bulk_mgr.done()
membership.roles.set([bdeRole])
elif membership.club.name == "Kfet":
membership.roles.set([kfetRole])
@timed @timed
def handle(self, *args, **kwargs): def handle(self, *args, **kwargs):
@ -268,4 +290,4 @@ class Command(ImportCommand):
self.load_map(kwargs["map"]) self.load_map(kwargs["map"])
self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"]) self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"])
self.adjust_roles() self.set_roles()

43
management/commands/syncsql.py Normal file
View File

@ -0,0 +1,43 @@
#!/usr/bin/env python3
from django.core.management.base import BaseCommand
from django.apps import apps
from django.db import connection
from polymorphic.models import PolymorphicModel
NO_SEQ = [
"Session",
"Token",
"WEIRole", # dirty fix
]
class Command(BaseCommand):
"""
Command to synchronise primary sequence of postgres after bulk insert of django.
"""
def add_arguments(self,parser):
parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized')
return parser
def handle(self, *args, **kwargs):
app_list = kwargs["apps"]
if len(app_list):
model_classes = list()
for app in app_list:
model_classes += apps.get_app_config(app).get_models()
else:
# no app specified, sync everything
model_classes = apps.get_models(include_auto_created=True)
db_names = [ m._meta.db_table for m in model_classes if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count()>1]
com = "BEGIN;\n"
for db_name in db_names:
com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "{db_name}";\n'
com += "COMMIT;"
print(com)
cur = connection.cursor()
cur.execute(com)
cur.close()

2
shell/tabularasa
View File

@ -1,4 +1,6 @@
#!/usr/bin/sh #!/usr/bin/sh
sudo service postgresql stop
sudo service postgresql start
sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"; sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'";
echo 'reset db'; echo 'reset db';
find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete