From 559be286b2988b199f86d56e51c20961dc233275 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 1 Jun 2020 17:54:49 +0200 Subject: [PATCH 1/5] add missing fields and fix bugs --- management/commands/import_account.py | 2 +- management/commands/import_transaction.py | 74 +++++++++++++++++------ 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/management/commands/import_account.py b/management/commands/import_account.py index 3a3fe11..9d0a2f4 100644 --- a/management/commands/import_account.py +++ b/management/commands/import_account.py @@ -82,7 +82,7 @@ class Command(ImportCommand): "balance": row['solde'], "last_negative": None, "is_active": True, - "display_image": "", + "display_image": "pic/default.png", "created_at": now() } if row["last_negatif"] is not None: diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py index 78c97db..d109bbf 100644 --- a/management/commands/import_transaction.py +++ b/management/commands/import_transaction.py @@ -8,28 +8,42 @@ import copy from django.utils.timezone import make_aware from django.db import transaction +from django.contrib.contenttypes.models import ContentType from note.models import (TemplateCategory, TransactionTemplate, Transaction, RecurrentTransaction, - SpecialTransaction + SpecialTransaction, + MembershipTransaction, ) from note.models import Note, NoteClub from activity.models import Guest, GuestTransaction -from member.models import Membership, MembershipTransaction +from member.models import Membership from ._import_utils import ImportCommand, BulkCreateManager, timed +# from member/fixtures/initial BDE_PK = 1 KFET_PK = 2 + +# from note/fixtures/initial NOTE_SPECIAL_CODE = { "espèce": 1, "carte": 2, "chèque": 3, "virement": 4, } +# from permission/fixtures/initial +BDE_ROLE_PK = 1 +KFET_ROLE_PK = 2 +CT = { + "RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"), + "SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"), + "MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"), + "GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"), +} def get_date_end(date_start): date_end = copy.deepcopy(date_start) @@ -78,22 +92,24 @@ class Command(ImportCommand): def _basic_transaction(self, row, obj_dict, child_dict): if len(row["description"]) > 255: - obj_dict["reason"] = obj_dict["reason"][:250]+"...)" + obj_dict["reason"] = obj_dict["reason"][:250] + "...)" return obj_dict, None, None def _template_transaction(self, row, obj_dict, child_dict): - if self.categories.get(row["categorie"]): - child_dict["category_id"] = self.categories[row["categorie"]] - elif "WEI" in row["description"]: - return obj_dict, None, None - elif self.buttons.get(row["description"]): + if self.buttons.get(row["description"]): child_dict["category_id"] = self.buttons[row["description"]][1] child_dict["template_id"] = self.buttons[row["description"]][0] + # elif self.categories.get(row["categorie"]): + # child_dict["category_id"] = self.categories[row["categorie"]] + elif "WEI" in row["description"]: + return obj_dict, None, None else: return obj_dict, None, None + obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"] return obj_dict, child_dict, RecurrentTransaction def _membership_transaction(self, row, obj_dict, child_dict, pk_membership): + obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"] obj_dict2 = obj_dict.copy() child_dict2 = child_dict.copy() child_dict2["membership_id"] = pk_membership @@ -103,6 +119,7 @@ class Command(ImportCommand): def _special_transaction(self, row, obj_dict, child_dict): # Some transaction uses BDE (idbde=0) as source or destination, # lets fix that. + obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"] field_id = "source_id" if row["type"] == "crédit" else "destination_id" if "espèce" in row["description"]: obj_dict[field_id] = 1 @@ -127,11 +144,12 @@ class Command(ImportCommand): def _guest_transaction(self, row, obj_dict, child_dict): # Currently GuestTransaction is related to a Guest. # This is not ideal and should be change to the Entry of this Guest. + obj_dict["polymorphic_ctype"] = CT["GuestTransaction"] m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"]) if m: first_name, last_name = m.group(2), m.group(3) guest_id = Guest.objects.filter(first_name__iexact=first_name, - last_name__iexact=last_name).first().pk + last_name__iexact=last_name).first().pk child_dict["guest_id"] = guest_id else: raise(f"Guest not Found {row['id']} {first_name}, last_name") @@ -165,35 +183,42 @@ class Command(ImportCommand): obj_dict = { "pk": pk_transaction, "destination_id": self.MAP_IDBDE[row["destinataire"]], + "polymorphic_ctype": None, "source_id": self.MAP_IDBDE[row["emetteur"]], - "created_at": date, "amount": row["montant"], + "created_at": date, + "destination_alias": "", + "invalidity_reason": None, "quantity": row["quantite"], "reason": row["description"], + "source_alias": "", "valid": row["valide"], } # for child transaction Models - child_dict = {"pk": obj_dict["pk"]} + child_dict = {"pk": pk_transaction} ttype = row["type"] + # Membership transaction detection and import if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"): note = Note.objects.get(pk=obj_dict["source_id"]) if isinstance(note, NoteClub): - child_transaction = None + child_transaction = None # don't bother register clubs else: user_id = note.user_id montant = obj_dict["amount"] - obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + (obj_dict0, + child_dict0, + child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership) bde_dict = { "pk": pk_membership, "user_id": user_id, - "club_id": KFET_PK, + "club_id": BDE_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": min(500, montant) } pk_membership += 1 pk_transaction += 1 - obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership) + obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict, pk_membership) # Kfet membership # BDE Membership obj_dict["pk"] = pk_transaction @@ -201,7 +226,7 @@ class Command(ImportCommand): kfet_dict = { "pk": pk_membership, "user_id": user_id, - "club_id": BDE_PK, + "club_id": KFET_PK, "date_start": date.date(), # Only date, not time "date_end": get_date_end(date.date()), "fee": max(montant - 500, 0), @@ -212,12 +237,12 @@ class Command(ImportCommand): pk_membership += 1 pk_transaction += 1 bulk_mgr.add( + Membership(**bde_dict), + Membership(**kfet_dict), Transaction(**obj_dict0), child_transaction(**child_dict0), Transaction(**obj_dict), child_transaction(**child_dict), - Membership(**bde_dict), - Membership(**kfet_dict), ) continue elif ttype == "bouton": @@ -226,7 +251,7 @@ class Command(ImportCommand): obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict) elif ttype == "invitation": obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict) - if ttype == "don" or ttype == "transfert": + elif ttype == "don" or ttype == "transfert": obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict) else: child_transaction = None @@ -236,6 +261,17 @@ class Command(ImportCommand): bulk_mgr.add(child_transaction(**child_dict)) pk_transaction += 1 bulk_mgr.done() + + def set_roles(self): + bulk_mgr = BulkCreateManager(chunk_size=10000) + membership_ids = Membership.objects.values_list('id',flat=True) + for m_id in membership_ids: + bulk_mgr.add( + Membership.roles.trough(membership_id=m_id,role_id=BDE_ROLE_PK), + Membership.roles.trough(membership_id=m_id,role_id=KFET_ROLE_PK), + ) + bulk_mgr.done() + @timed def handle(self, *args, **kwargs): # default args, provided by ImportCommand. From 9e8d0901d1eae61b81076d8284126db103dfd694 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 1 Jun 2020 17:55:03 +0200 Subject: [PATCH 2/5] restart postgres to disconnect everybody --- shell/tabularasa | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shell/tabularasa b/shell/tabularasa index 0a81c5f..b6ccbcd 100755 --- a/shell/tabularasa +++ b/shell/tabularasa @@ -1,4 +1,6 @@ #!/usr/bin/sh +sudo service postgresql stop +sudo service postgresql start sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"; echo 'reset db'; find apps/ -path "*/migrations/*.py*" -not -name "__init__.py" -delete From 71ec40cd9580699832cfb346049f1ac2232751d6 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 1 Jun 2020 17:55:24 +0200 Subject: [PATCH 3/5] automatically gives su roles to developers --- management/commands/import_nk15.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index b0a8236..f85204a 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -12,7 +12,7 @@ class Command(BaseCommand): def handle(self, *args, **kwargs): subprocess.call("./apps/scripts/shell/tabularasa") - call_command('import_account', alias=True, chunk=1000, save = "map.json") - call_command('import_activities', chunk=100, map="map.json") - call_command('import_transaction', chunk=10000, buttons=True, map="map.json") -# + call_command('import_account', alias=True, chunk=5000, save = "map.json") + call_command('import_activities', chunk=5000, map="map.json") + call_command('import_transaction', chunk=5000, buttons=True, map="map.json") + call_command('make_su', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant') From 7d9599d4d80a698352fe27f1403bc5550e2c69e0 Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Mon, 1 Jun 2020 22:27:41 +0200 Subject: [PATCH 4/5] typo and forget to call function --- management/commands/import_nk15.py | 2 +- management/commands/import_transaction.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index f85204a..519a1b5 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -15,4 +15,4 @@ class Command(BaseCommand): call_command('import_account', alias=True, chunk=5000, save = "map.json") call_command('import_activities', chunk=5000, map="map.json") call_command('import_transaction', chunk=5000, buttons=True, map="map.json") - call_command('make_su', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant') + call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant') diff --git a/management/commands/import_transaction.py b/management/commands/import_transaction.py index d109bbf..0d7d9e8 100644 --- a/management/commands/import_transaction.py +++ b/management/commands/import_transaction.py @@ -267,10 +267,10 @@ class Command(ImportCommand): membership_ids = Membership.objects.values_list('id',flat=True) for m_id in membership_ids: bulk_mgr.add( - Membership.roles.trough(membership_id=m_id,role_id=BDE_ROLE_PK), - Membership.roles.trough(membership_id=m_id,role_id=KFET_ROLE_PK), + Membership.roles.through(membership_id=m_id,role_id=BDE_ROLE_PK), + Membership.roles.through(membership_id=m_id,role_id=KFET_ROLE_PK), ) - bulk_mgr.done() + bulk_mgr.done() @timed def handle(self, *args, **kwargs): @@ -284,3 +284,4 @@ class Command(ImportCommand): self.load_map(kwargs["map"]) self.import_buttons(cur, kwargs["chunk"]) self.import_transaction(cur, kwargs["chunk"], 0) + self.set_roles() From dc1daf0a2ddd6a74be0f2c06473a6ad16596d10b Mon Sep 17 00:00:00 2001 From: Pierre-antoine Comby Date: Tue, 2 Jun 2020 10:46:08 +0200 Subject: [PATCH 5/5] add command to synchronize db sequences --- management/commands/import_nk15.py | 1 + management/commands/syncsql.py | 43 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 management/commands/syncsql.py diff --git a/management/commands/import_nk15.py b/management/commands/import_nk15.py index 519a1b5..12d7164 100644 --- a/management/commands/import_nk15.py +++ b/management/commands/import_nk15.py @@ -16,3 +16,4 @@ class Command(BaseCommand): call_command('import_activities', chunk=5000, map="map.json") call_command('import_transaction', chunk=5000, buttons=True, map="map.json") call_command('make_su','-sS', 'Coq', 'erdnaxe', 'PAC', 'Pollion', 'ÿnérant') + call_command('syncsql') diff --git a/management/commands/syncsql.py b/management/commands/syncsql.py new file mode 100644 index 0000000..92a908d --- /dev/null +++ b/management/commands/syncsql.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 + +from django.core.management.base import BaseCommand +from django.apps import apps +from django.db import connection + + +from polymorphic.models import PolymorphicModel + +NO_SEQ = [ + "Session", + "Token", + "WEIRole", # dirty fix +] + +class Command(BaseCommand): + """ + Command to synchronise primary sequence of postgres after bulk insert of django. + """ + + def add_arguments(self,parser): + parser.add_argument('apps', type=str,nargs='*',help='applications which table would be resynchronized') + return parser + + def handle(self, *args, **kwargs): + app_list = kwargs["apps"] + if len(app_list): + model_classes = list() + for app in app_list: + model_classes += apps.get_app_config(app).get_models() + else: + # no app specified, sync everything + model_classes = apps.get_models(include_auto_created=True) + + db_names = [ m._meta.db_table for m in model_classes if m.__base__.__base__ is not PolymorphicModel and m.__name__ not in NO_SEQ and m.objects.count()>1] + com = "BEGIN;\n" + for db_name in db_names: + com += f'SELECT setval(pg_get_serial_sequence(\'"{db_name}"\',\'id\'), coalesce(max("id"), 1), max("id") IS NOT null) FROM "{db_name}";\n' + com += "COMMIT;" + print(com) + cur = connection.cursor() + cur.execute(com) + cur.close()