everything is imported, without server error
This commit is contained in:
parent
5af336fff3
commit
0ba656d5e0
|
@ -15,6 +15,15 @@ from ._import_utils import ImportCommand, BulkCreateManager, timed
|
||||||
|
|
||||||
MAP_ACTIVITY = dict()
|
MAP_ACTIVITY = dict()
|
||||||
|
|
||||||
|
CLUB_RELOU = [
|
||||||
|
0, # BDE
|
||||||
|
4771, # Kataclist
|
||||||
|
5162, # Assurance BDE ?!
|
||||||
|
5164, #S & L
|
||||||
|
625, #Aspique
|
||||||
|
5154, #Frekens
|
||||||
|
]
|
||||||
|
|
||||||
class Command(ImportCommand):
|
class Command(ImportCommand):
|
||||||
"""
|
"""
|
||||||
Import command for Activities Base Data (Comptes, and Aliases)
|
Import command for Activities Base Data (Comptes, and Aliases)
|
||||||
|
@ -54,7 +63,7 @@ class Command(ImportCommand):
|
||||||
"date_start": make_aware(row["debut"]),
|
"date_start": make_aware(row["debut"]),
|
||||||
"date_end": make_aware(row["fin"]),
|
"date_end": make_aware(row["fin"]),
|
||||||
"valid": row["validepar"] is not None,
|
"valid": row["validepar"] is not None,
|
||||||
"open": row["ouvert"], # Should always be False
|
"open": False,
|
||||||
}
|
}
|
||||||
# WARNING: Fields lieu, liste, listeimprimee are missing
|
# WARNING: Fields lieu, liste, listeimprimee are missing
|
||||||
MAP_ACTIVITY[row["id"]] = pk_activity
|
MAP_ACTIVITY[row["id"]] = pk_activity
|
||||||
|
@ -70,6 +79,8 @@ class Command(ImportCommand):
|
||||||
n = cur.rowcount
|
n = cur.rowcount
|
||||||
for idx, row in enumerate(cur):
|
for idx, row in enumerate(cur):
|
||||||
self.update_line(idx, n, f"{row['nom']} {row['prenom']}")
|
self.update_line(idx, n, f"{row['nom']} {row['prenom']}")
|
||||||
|
if row["responsable"] in CLUB_RELOU:
|
||||||
|
row["responsable"] = 3508
|
||||||
obj_dict = {
|
obj_dict = {
|
||||||
"pk": row["id"],
|
"pk": row["id"],
|
||||||
"activity_id": MAP_ACTIVITY[row["activite"]],
|
"activity_id": MAP_ACTIVITY[row["activite"]],
|
||||||
|
@ -88,11 +99,13 @@ class Command(ImportCommand):
|
||||||
n = cur.rowcount
|
n = cur.rowcount
|
||||||
for idx, row in enumerate(cur):
|
for idx, row in enumerate(cur):
|
||||||
self.update_line(idx, n, f"{row['idbde']} {row['responsable']}")
|
self.update_line(idx, n, f"{row['idbde']} {row['responsable']}")
|
||||||
|
if row["idbde"] in CLUB_RELOU:
|
||||||
|
row["idbde"] = 3508
|
||||||
obj_dict = {
|
obj_dict = {
|
||||||
"activity_id": MAP_ACTIVITY[row["activite"]],
|
"activity_id": MAP_ACTIVITY[row["activite"]],
|
||||||
"time": make_aware(row["heure_entree"]),
|
"time": make_aware(row["heure_entree"]),
|
||||||
"note_id": self.MAP_IDBDE[row["responsable"] if row['est_invite'] else row["idbde"]],
|
"note_id": self.MAP_IDBDE[row["responsable"] if row['est_invite'] else row["idbde"]],
|
||||||
"guest_id": self.MAP_IDBDE[row["idbde"]] if row['est_invite'] else None,
|
"guest_id": row["idbde"] if row['est_invite'] else None,
|
||||||
}
|
}
|
||||||
bulk_mgr.add(Entry(**obj_dict))
|
bulk_mgr.add(Entry(**obj_dict))
|
||||||
bulk_mgr.done()
|
bulk_mgr.done()
|
||||||
|
|
|
@ -15,7 +15,7 @@ from note.models import (TemplateCategory,
|
||||||
RecurrentTransaction,
|
RecurrentTransaction,
|
||||||
SpecialTransaction
|
SpecialTransaction
|
||||||
)
|
)
|
||||||
from note.models import Note
|
from note.models import Note, NoteClub
|
||||||
from activity.models import Guest, GuestTransaction
|
from activity.models import Guest, GuestTransaction
|
||||||
|
|
||||||
from member.models import Membership, MembershipTransaction
|
from member.models import Membership, MembershipTransaction
|
||||||
|
@ -33,10 +33,9 @@ NOTE_SPECIAL_CODE = {
|
||||||
|
|
||||||
def get_date_end(date_start):
|
def get_date_end(date_start):
|
||||||
date_end = copy.deepcopy(date_start)
|
date_end = copy.deepcopy(date_start)
|
||||||
if date_start > 8:
|
if date_start.month > 8:
|
||||||
date_end.year = date_start + 1
|
date_end = date_start.replace(year=date_start.year+1)
|
||||||
date_end.month = 9
|
date_end = date_end.replace(month=9, day=30)
|
||||||
date_end.day = 30
|
|
||||||
return date_end
|
return date_end
|
||||||
|
|
||||||
|
|
||||||
|
@ -50,39 +49,98 @@ class Command(ImportCommand):
|
||||||
parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
|
parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
|
||||||
|
|
||||||
@timed
|
@timed
|
||||||
@transaction.atomic
|
|
||||||
def import_buttons(self, cur, chunk_size):
|
def import_buttons(self, cur, chunk_size):
|
||||||
categories = dict()
|
self.categories = dict()
|
||||||
buttons = dict()
|
self.buttons = dict()
|
||||||
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
||||||
cur.execute("SELECT * FROM boutons;")
|
cur.execute("SELECT * FROM boutons;")
|
||||||
n = cur.rowcount
|
n = cur.rowcount
|
||||||
pk_category = 1
|
|
||||||
for idx, row in enumerate(cur):
|
for idx, row in enumerate(cur):
|
||||||
self.update_line(idx, n, row["label"])
|
self.update_line(idx, n, row["label"])
|
||||||
if row["categorie"] not in categories:
|
if row["categorie"] not in self.categories:
|
||||||
bulk_mgr.add(TemplateCategory(pk=pk_category, name=row["categorie"]))
|
cat = TemplateCategory.objects.create(name=row["categorie"])
|
||||||
pk_category += 1
|
cat.save()
|
||||||
categories[row["categorie"]] = pk_category
|
self.categories[row["categorie"]] = cat.pk
|
||||||
obj_dict = {
|
obj_dict = {
|
||||||
"pk": row["id"],
|
"pk": row["id"],
|
||||||
"name": row["label"],
|
"name": row["label"],
|
||||||
"amount": row["montant"],
|
"amount": row["montant"],
|
||||||
"destination_id": self.MAP_IDBDE[row["destinataire"]],
|
"destination_id": self.MAP_IDBDE[row["destinataire"]],
|
||||||
"category_id": categories[row["categorie"]],
|
"category_id": self.categories[row["categorie"]],
|
||||||
"display": row["affiche"],
|
"display": row["affiche"],
|
||||||
"description": row["description"],
|
"description": row["description"],
|
||||||
}
|
}
|
||||||
if row["label"] in buttons:
|
if row["label"] in self.buttons:
|
||||||
obj_dict["label"] = f"{obj_dict['label']}_{obj_dict['destination_id']}"
|
obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
|
||||||
bulk_mgr.add(TransactionTemplate(**obj_dict))
|
bulk_mgr.add(TransactionTemplate(**obj_dict))
|
||||||
buttons[obj_dict["label"]] = row["id"]
|
self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
|
||||||
bulk_mgr.done()
|
bulk_mgr.done()
|
||||||
return buttons, categories
|
|
||||||
|
def _basic_transaction(self, row, obj_dict, child_dict):
|
||||||
|
if len(row["description"]) > 255:
|
||||||
|
obj_dict["reason"] = obj_dict["reason"][:250]+"...)"
|
||||||
|
return obj_dict, None, None
|
||||||
|
|
||||||
|
def _template_transaction(self, row, obj_dict, child_dict):
|
||||||
|
if self.categories.get(row["categorie"]):
|
||||||
|
child_dict["category_id"] = self.categories[row["categorie"]]
|
||||||
|
elif "WEI" in row["description"]:
|
||||||
|
return obj_dict, None, None
|
||||||
|
elif self.buttons.get(row["description"]):
|
||||||
|
child_dict["category_id"] = self.buttons[row["description"]][1]
|
||||||
|
child_dict["template_id"] = self.buttons[row["description"]][0]
|
||||||
|
else:
|
||||||
|
return obj_dict, None, None
|
||||||
|
return obj_dict, child_dict, RecurrentTransaction
|
||||||
|
|
||||||
|
def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
|
||||||
|
obj_dict2 = obj_dict.copy()
|
||||||
|
child_dict2 = child_dict.copy()
|
||||||
|
child_dict2["membership_id"] = pk_membership
|
||||||
|
|
||||||
|
return obj_dict2, child_dict2, MembershipTransaction
|
||||||
|
|
||||||
|
def _special_transaction(self, row, obj_dict, child_dict):
|
||||||
|
# Some transaction uses BDE (idbde=0) as source or destination,
|
||||||
|
# lets fix that.
|
||||||
|
field_id = "source_id" if row["type"] == "crédit" else "destination_id"
|
||||||
|
if "espèce" in row["description"]:
|
||||||
|
obj_dict[field_id] = 1
|
||||||
|
elif "carte" in row["description"]:
|
||||||
|
obj_dict[field_id] = 2
|
||||||
|
elif "cheques" in row["description"]:
|
||||||
|
obj_dict[field_id] = 3
|
||||||
|
elif "virement" in row["description"]:
|
||||||
|
obj_dict[field_id] = 4
|
||||||
|
# humans and clubs have always the biggest id
|
||||||
|
actor_pk = max(row["destinataire"], row["emetteur"])
|
||||||
|
actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
|
||||||
|
# custom fields of SpecialTransaction
|
||||||
|
if actor.__class__.__name__ == "NoteUser":
|
||||||
|
child_dict["first_name"] = actor.user.first_name
|
||||||
|
child_dict["last_name"] = actor.user.last_name
|
||||||
|
else:
|
||||||
|
child_dict["first_name"] = actor.club.name
|
||||||
|
child_dict["last_name"] = actor.club.name
|
||||||
|
return obj_dict, child_dict, SpecialTransaction
|
||||||
|
|
||||||
|
def _guest_transaction(self, row, obj_dict, child_dict):
|
||||||
|
# Currently GuestTransaction is related to a Guest.
|
||||||
|
# This is not ideal and should be change to the Entry of this Guest.
|
||||||
|
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
|
||||||
|
if m:
|
||||||
|
first_name, last_name = m.group(2), m.group(3)
|
||||||
|
guest_id = Guest.objects.filter(first_name__iexact=first_name,
|
||||||
|
last_name__iexact=last_name).first().pk
|
||||||
|
child_dict["guest_id"] = guest_id
|
||||||
|
else:
|
||||||
|
raise(f"Guest not Found {row['id']} {first_name}, last_name")
|
||||||
|
|
||||||
|
return obj_dict, child_dict, GuestTransaction
|
||||||
|
|
||||||
@timed
|
@timed
|
||||||
@transaction.atomic
|
@transaction.atomic
|
||||||
def import_transaction(self, cur, chunk_size, idmin, buttons, categories):
|
def import_transaction(self, cur, chunk_size, idmin):
|
||||||
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
|
||||||
cur.execute(
|
cur.execute(
|
||||||
f"SELECT t.date AS transac_date, t.type, t.emetteur,\
|
f"SELECT t.date AS transac_date, t.type, t.emetteur,\
|
||||||
|
@ -117,89 +175,68 @@ class Command(ImportCommand):
|
||||||
# for child transaction Models
|
# for child transaction Models
|
||||||
child_dict = {"pk": obj_dict["pk"]}
|
child_dict = {"pk": obj_dict["pk"]}
|
||||||
ttype = row["type"]
|
ttype = row["type"]
|
||||||
if ttype == "don" or ttype == "transfert":
|
if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
|
||||||
child_transaction = None
|
note = Note.objects.get(pk=obj_dict["source_id"])
|
||||||
|
if isinstance(note, NoteClub):
|
||||||
|
child_transaction = None
|
||||||
|
else:
|
||||||
|
user_id = note.user_id
|
||||||
|
montant = obj_dict["amount"]
|
||||||
|
obj_dict0, child_dict0, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
|
||||||
|
bde_dict = {
|
||||||
|
"pk": pk_membership,
|
||||||
|
"user_id": user_id,
|
||||||
|
"club_id": KFET_PK,
|
||||||
|
"date_start": date.date(), # Only date, not time
|
||||||
|
"date_end": get_date_end(date.date()),
|
||||||
|
"fee": min(500, montant)
|
||||||
|
}
|
||||||
|
pk_membership += 1
|
||||||
|
pk_transaction += 1
|
||||||
|
obj_dict, child_dict, child_transaction = self._membership_transaction(row, obj_dict, child_dict,pk_membership)
|
||||||
|
# Kfet membership
|
||||||
|
# BDE Membership
|
||||||
|
obj_dict["pk"] = pk_transaction
|
||||||
|
child_dict["pk"] = pk_transaction
|
||||||
|
kfet_dict = {
|
||||||
|
"pk": pk_membership,
|
||||||
|
"user_id": user_id,
|
||||||
|
"club_id": BDE_PK,
|
||||||
|
"date_start": date.date(), # Only date, not time
|
||||||
|
"date_end": get_date_end(date.date()),
|
||||||
|
"fee": max(montant - 500, 0),
|
||||||
|
}
|
||||||
|
obj_dict0["amount"] = bde_dict["fee"]
|
||||||
|
obj_dict["amount"] = kfet_dict["fee"]
|
||||||
|
# BDE membership Transaction is inserted before the Kfet membershipTransaction
|
||||||
|
pk_membership += 1
|
||||||
|
pk_transaction += 1
|
||||||
|
bulk_mgr.add(
|
||||||
|
Transaction(**obj_dict0),
|
||||||
|
child_transaction(**child_dict0),
|
||||||
|
Transaction(**obj_dict),
|
||||||
|
child_transaction(**child_dict),
|
||||||
|
Membership(**bde_dict),
|
||||||
|
Membership(**kfet_dict),
|
||||||
|
)
|
||||||
|
continue
|
||||||
elif ttype == "bouton":
|
elif ttype == "bouton":
|
||||||
child_transaction = RecurrentTransaction
|
obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
|
||||||
child_dict["category_id"] = categories.get(row["categorie"], categories["Autre"])
|
|
||||||
child_dict["template_id"] = buttons[row["description"]]
|
|
||||||
elif ttype == "crédit" or ttype == "retrait":
|
elif ttype == "crédit" or ttype == "retrait":
|
||||||
child_transaction = SpecialTransaction
|
obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
|
||||||
# Some transaction uses BDE (idbde=0) as source or destination,
|
|
||||||
# lets fix that.
|
|
||||||
field_id = "source_id" if ttype == "crédit" else "destination_id"
|
|
||||||
if "espèce" in row["description"]:
|
|
||||||
obj_dict[field_id] = 1
|
|
||||||
elif "carte" in row["description"]:
|
|
||||||
obj_dict[field_id] = 2
|
|
||||||
elif "cheques" in row["description"]:
|
|
||||||
obj_dict[field_id] = 3
|
|
||||||
elif "virement" in row["description"]:
|
|
||||||
obj_dict[field_id] = 4
|
|
||||||
# humans and clubs have always the biggest id
|
|
||||||
actor_pk = max(row["destinataire"], row["emetteur"])
|
|
||||||
actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
|
|
||||||
# custom fields of SpecialTransaction
|
|
||||||
if actor.__class__.__name__ == "NoteUser":
|
|
||||||
child_dict["first_name"] = actor.user.first_name
|
|
||||||
child_dict["last_name"] = actor.user.last_name
|
|
||||||
else:
|
|
||||||
child_dict["first_name"] = actor.club.name
|
|
||||||
child_dict["last_name"] = actor.club.name
|
|
||||||
elif ttype == "adhésion" and row["valide"]:
|
|
||||||
child_transaction = MembershipTransaction
|
|
||||||
# Kfet membership
|
|
||||||
montant = row["montant"]
|
|
||||||
obj_dict["amount"] = min(500, montant)
|
|
||||||
child_dict["membership_id"] = pk_membership
|
|
||||||
kfet_dict = {
|
|
||||||
"pk": pk_membership,
|
|
||||||
"user": self.MAP_IDBDE[row["idbde"]],
|
|
||||||
"club": KFET_PK,
|
|
||||||
"date_start": row["date"].date(), # Only date, not time
|
|
||||||
"date_end": get_date_end(row["date"].date()),
|
|
||||||
"fee": min(500, montant)
|
|
||||||
}
|
|
||||||
|
|
||||||
pk_membership += 1
|
|
||||||
pk_transaction += 1
|
|
||||||
# BDE Membership
|
|
||||||
obj_dict2 = obj_dict.copy()
|
|
||||||
child_dict2 = dict()
|
|
||||||
obj_dict2["pk"] = pk_transaction
|
|
||||||
obj_dict2["amount"] = max(montant - 500, 0)
|
|
||||||
child_dict2["pk"] = pk_transaction
|
|
||||||
bde_dict = {
|
|
||||||
"pk": pk_membership,
|
|
||||||
"user": self.MAP_IDBDE[row["idbde"]],
|
|
||||||
"club": BDE_PK,
|
|
||||||
"date_start": row["date"].date(), # Only date, not time
|
|
||||||
"date_end": get_date_end(row["date"].date()),
|
|
||||||
"fee": max(montant - 500, 0),
|
|
||||||
}
|
|
||||||
pk_membership += 1
|
|
||||||
# BDE membership Transaction is inserted before the Kfet membershipTransaction
|
|
||||||
bulk_mgr.add(
|
|
||||||
Transaction(**obj_dict2),
|
|
||||||
child_transaction(**child_dict2),
|
|
||||||
Membership(**bde_dict),
|
|
||||||
Membership(**kfet_dict),
|
|
||||||
)
|
|
||||||
elif ttype == "invitation":
|
elif ttype == "invitation":
|
||||||
child_transaction = GuestTransaction
|
obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
|
||||||
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
|
if ttype == "don" or ttype == "transfert":
|
||||||
if m:
|
obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
|
||||||
first_name, last_name = m.groups(1), m.groups(2)
|
else:
|
||||||
guest_id = Guest.object.filter(first_name__iexact=first_name,
|
child_transaction = None
|
||||||
last_name__iexact=last_name).first().pk
|
# create base transaction object and typed one
|
||||||
child_dict["guest_id"] = guest_id
|
bulk_mgr.add(Transaction(**obj_dict))
|
||||||
else:
|
if child_transaction is not None:
|
||||||
raise(f"Guest not Found {row['id']} {first_name}, last_name" )
|
bulk_mgr.add(child_transaction(**child_dict))
|
||||||
|
|
||||||
bulk_mgr.add(Transaction(**obj_dict),
|
|
||||||
child_transaction(**child_dict))
|
|
||||||
pk_transaction += 1
|
pk_transaction += 1
|
||||||
|
bulk_mgr.done()
|
||||||
|
@timed
|
||||||
def handle(self, *args, **kwargs):
|
def handle(self, *args, **kwargs):
|
||||||
# default args, provided by ImportCommand.
|
# default args, provided by ImportCommand.
|
||||||
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
|
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
|
||||||
|
@ -208,8 +245,6 @@ class Command(ImportCommand):
|
||||||
cur = conn.cursor(cursor_factory=pge.DictCursor)
|
cur = conn.cursor(cursor_factory=pge.DictCursor)
|
||||||
|
|
||||||
if kwargs["map"]:
|
if kwargs["map"]:
|
||||||
self.load(kwargs["map"])
|
self.load_map(kwargs["map"])
|
||||||
|
|
||||||
self.import_buttons(cur, kwargs["chunk"])
|
self.import_buttons(cur, kwargs["chunk"])
|
||||||
|
self.import_transaction(cur, kwargs["chunk"], 0)
|
||||||
self.import_transaction(cur, kwargs["chunk"])
|
|
||||||
|
|
Loading…
Reference in New Issue