diff --git a/management/commands/_import_utils.py b/management/commands/_import_utils.py index 70a40cd..449609d 100644 --- a/management/commands/_import_utils.py +++ b/management/commands/_import_utils.py @@ -44,7 +44,7 @@ class ImportCommand(BaseCommand): n = str(n) total = str(total) n.rjust(len(total)) - print(f"\r ({n}/{total}) {content:10.10}", end="") + print(f"\r ({n}/{total}) {content:16.16}", end="") def create_parser(self, prog_name, subcommand, **kwargs): parser = super().create_parser(prog_name, subcommand, **kwargs) diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index 17084e9..5990482 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -30,4 +30,6 @@ class Command(ImportCommand): kwargs["map"] = "map.json" kwargs["buttons"] = True call_command('import_transaction', **kwargs) -# + + call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant') + call_command('syncsql') diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py index 718a4c6..1ee2ee2 100644 --- a/management/commands/import_transaction.py +++ b/management/commands/import_transaction.py @@ -8,12 +8,14 @@ import copy from django.utils.timezone import make_aware from django.db import transaction +from django.contrib.contenttypes.models import ContentType from note.models import (TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, - SpecialTransaction + SpecialTransaction, + MembershipTransaction, ) from note.models import Note, NoteClub from activity.models import Guest, GuestTransaction @@ -21,15 +23,27 @@ from activity.models import Guest, GuestTransaction from member.models import Membership, MembershipTransaction, Role from ._import_utils import ImportCommand, BulkCreateManager, timed +# from member/fixtures/initial BDE_PK = 1 KFET_PK = 2 + +# from note/fixtures/initial NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } +# from permission/fixtures/initial +BDE_ROLE_PK = 1 +KFET_ROLE_PK = 2 +CT = { + "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"), + "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"), + "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"), + "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"), +} def get_date_end(date_start): date_end = copy.deepcopy(date_start) @@ -79,22 +93,24 @@ class Command(ImportCommand): def _basic_transaction(self, row, obj_dict, child_dict): if len(row["description"]) > 255: - obj_dict["reason"] = obj_dict["reason"][:250]+"...)" + obj_dict["reason"] = obj_dict["reason"][:250] + "...)" return obj_dict, None, None def _template_transaction(self, row, obj_dict, child_dict): - if self.categories.get(row["categorie"]): - child_dict["category_id"] = self.categories[row["categorie"]] - elif "WEI" in row["description"]: - return obj_dict, None, None - elif self.buttons.get(row["description"]): + if self.buttons.get(row["description"]): child_dict["category_id"] = self.buttons[row["description"]][1] child_dict["template_id"] = self.buttons[row["description"]][0] + # elif self.categories.get(row["categorie"]): + # child_dict["category_id"] = self.categories[row["categorie"]] + elif "WEI" in row["description"]: + return obj_dict, None, None else: return obj_dict, None, None + obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"] return obj_dict, child_dict, RecurrentTransaction def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): + obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"] obj_dict2 = obj_dict.copy() child_dict2 = child_dict.copy() child_dict2["membership_id"] = pk_membership @@ -104,6 +120,7 @@ class Command(ImportCommand): def _special_transaction(self, row, obj_dict, child_dict): # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. + obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"] field_id = "source_id" if row["type"] == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 @@ -128,13 +145,14 @@ class Command(ImportCommand): def _guest_transaction(self, row, obj_dict, child_dict): # Currently GuestTransaction is related to a Guest. # This is not ideal and should be change to the Entry of this Guest. + obj_dict["polymorphic_ctype"] = CT["GuestTransaction"] m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: first_name, last_name = m.group(2), m.group(3) if first_name == "Marion" and last_name == "Bizu Pose": first_name, last_name = "Marion Bizu", "Pose" guest_id = Guest.objects.filter(first_name__iexact=first_name, - last_name__iexact=last_name).first().pk + last_name__iexact=last_name).first().pk child_dict["guest_id"] = guest_id else: raise(f"Guest not Found {row['id']} {first_name}, last_name") @@ -171,35 +189,42 @@ class Command(ImportCommand): obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], + "polymorphic_ctype": None, "source_id": self.MAP_IDBDE[row["emetteur"]], - "created_at": date, "amount": row["montant"], + "created_at": date, + "destination_alias": "", + "invalidity_reason": None, "quantity": row["quantite"], "reason": row["description"], + "source_alias": "", "valid": row["valide"], } # for child transaction Models - child_dict = {"pk": obj_dict["pk"]} + child_dict = {"pk": pk_transaction} ttype = row["type"] + # Membership transaction detection and import if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): - child_transaction = None + child_transaction = None # don't bother register clubs else: user_id = note.user_id montant = obj_dict["amount"] - obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + (obj_dict0, + child_dict0, + child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership) bde_dict = { "pk": pk_membership, "user_id": user_id, - "club_id": KFET_PK, + "club_id": BDE_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 - obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict, pk_membership) # Kfet membership # BDE Membership obj_dict["pk"] = pk_transaction @@ -207,7 +232,7 @@ class Command(ImportCommand): kfet_dict = { "pk": pk_membership, "user_id": user_id, - "club_id": BDE_PK, + "club_id": KFET_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": max(montant - 500, 0), @@ -218,12 +243,12 @@ class Command(ImportCommand): pk_membership += 1 pk_transaction += 1 bulk_mgr.add( + Membership(**bde_dict), + Membership(**kfet_dict), Transaction(**obj_dict0), child_transaction(**child_dict0), Transaction(**obj_dict), child_transaction(**child_dict), - Membership(**bde_dict), - Membership(**kfet_dict), ) continue elif ttype == "bouton": @@ -232,7 +257,7 @@ class Command(ImportCommand): obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) - if ttype == "don" or ttype == "transfert": + elif ttype == "don" or ttype == "transfert": obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) else: child_transaction = None @@ -243,18 +268,15 @@ class Command(ImportCommand): pk_transaction += 1 bulk_mgr.done() - @timed - def adjust_roles(self): - bdeRole = Role.objects.get(name="Adhérent BDE") - kfetRole = Role.objects.get(name="Adhérent Kfet") - memberships = Membership.objects.all() - n = len(memberships) - for idx, membership in enumerate(memberships): - self.update_line(idx, n, membership.user.username) - if membership.club.name == "BDE": - membership.roles.set([bdeRole]) - elif membership.club.name == "Kfet": - membership.roles.set([kfetRole]) + def set_roles(self): + bulk_mgr = BulkCreateManager(chunk_size=10000) + membership_ids = Membership.objects.values_list('id',flat=True) + for m_id in membership_ids: + bulk_mgr.add( + Membership.roles.through(membership_id=m_id,role_id=BDE_ROLE_PK), + Membership.roles.through(membership_id=m_id,role_id=KFET_ROLE_PK), + ) + bulk_mgr.done() @timed def handle(self, *args, **kwargs): @@ -268,4 +290,4 @@ class Command(ImportCommand): self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"]) self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"]) - self.adjust_roles() + self.set_roles() diff --git a/management/commands/syncsql.py b/management/commands/syncsql.py new file mode 100644 index 0000000..92a908d --- /dev/null +++ b/management/commands/syncsql.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from django.core.management.base import BaseCommand +from django.apps import apps +from django.db import connection + + +from polymorphic.models import PolymorphicModel + +NO_SEQ = [ + "Session", + "Token", + "WEIRole", # dirty fix +] + +class Command(BaseCommand): + """ + Command to synchronise primary sequence of postgres after bulk insert of django. + """ + + def add_arguments(self,parser): + parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized') + return parser + + def handle(self, *args, **kwargs): + app_list = kwargs["apps"] + if len(app_list): + model_classes = list() + for app in app_list: + model_classes += apps.get_app_config(app).get_models() + else: + # no app specified, sync everything + model_classes = apps.get_models(include_auto_created=True) + + db_names = [ m._meta.db_table for m in model_classes if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count()>1] + com = "BEGIN;\n" + for db_name in db_names: + com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "{db_name}";\n' + com += "COMMIT;" + print(com) + cur = connection.cursor() + cur.execute(com) + cur.close() diff --git a/shell/tabularasa b/shell/tabularasa index 0a81c5f..b6ccbcd 100755 --- a/shell/tabularasa +++ b/shell/tabularasa @@ -1,4 +1,6 @@ #!/usr/bin/sh +sudo service postgresql stop +sudo service postgresql start sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"; echo 'reset db'; find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete