nk20-scripts/management/commands/import_transaction.py

417 lines
17 KiB
Python
Raw Normal View History

2020-07-25 19:57:46 +00:00
# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
# SPDX-License-Identifier: GPL-3.0-or-later
import re
import psycopg2 as pg
import psycopg2.extras as pge
import pytz
import datetime
import copy
2020-08-07 11:17:17 +00:00
from django.contrib.auth.models import User
from django.utils.timezone import make_aware
from django.db import transaction
2020-06-01 15:54:49 +00:00
from django.contrib.contenttypes.models import ContentType
from note.models import (TemplateCategory,
TransactionTemplate,
Transaction,
RecurrentTransaction,
2020-06-01 15:54:49 +00:00
SpecialTransaction,
MembershipTransaction,
)
from note.models import Note, NoteClub
2020-07-23 05:21:29 +00:00
from activity.models import Guest, GuestTransaction, Entry
2020-07-23 05:21:29 +00:00
from member.models import Membership
2020-08-07 11:17:17 +00:00
from treasury.models import Remittance, SpecialTransactionProxy, SogeCredit
from ._import_utils import ImportCommand, BulkCreateManager, timed
2020-07-25 23:01:17 +00:00
MAP_TRANSACTION = dict()
MAP_REMITTANCE = dict()
2020-06-01 15:54:49 +00:00
# from member/fixtures/initial
BDE_PK = 1
KFET_PK = 2
2020-06-01 15:54:49 +00:00
# from note/fixtures/initial
NOTE_SPECIAL_CODE = {
"espèce": 1,
"carte": 2,
"chèque": 3,
"virement": 4,
}
2020-06-01 15:54:49 +00:00
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2
2020-06-01 15:54:49 +00:00
CT = {
"RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
"SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
"MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
"GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}
def get_date_end(date_start):
date_end = copy.deepcopy(date_start)
if date_start.month >= 8:
date_end = date_start.replace(year=date_start.year + 1)
date_end = date_end.replace(month=9, day=30)
return date_end
class Command(ImportCommand):
"""
Import command for People base data (Comptes, and Aliases)
"""
def add_arguments(self, parser):
parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
2020-07-26 22:16:24 +00:00
parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
"don't save them")
@timed
2020-07-21 23:28:28 +00:00
def import_buttons(self, cur, chunk_size, import_buttons):
self.categories = dict()
self.buttons = dict()
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute("SELECT * FROM boutons;")
n = cur.rowcount
for idx, row in enumerate(cur):
self.update_line(idx, n, row["label"])
if row["categorie"] not in self.categories:
2020-07-21 23:28:28 +00:00
cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
cat.save()
self.categories[row["categorie"]] = cat.pk
obj_dict = {
"pk": row["id"],
"name": row["label"],
"amount": row["montant"],
"destination_id": self.MAP_IDBDE[row["destinataire"]],
"category_id": self.categories[row["categorie"]],
"display": row["affiche"],
"description": row["description"],
}
if row["label"] in self.buttons:
obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
2020-07-21 23:28:28 +00:00
if import_buttons:
bulk_mgr.add(TransactionTemplate(**obj_dict))
self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
bulk_mgr.done()
def _basic_transaction(self, row, obj_dict, child_dict):
if len(row["description"]) > 255:
2020-06-01 15:54:49 +00:00
obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
return obj_dict, None, None
def _template_transaction(self, row, obj_dict, child_dict):
2020-06-01 15:54:49 +00:00
if self.buttons.get(row["description"]):
child_dict["template_id"] = self.buttons[row["description"]][0]
2020-06-01 15:54:49 +00:00
# elif self.categories.get(row["categorie"]):
# child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
else:
return obj_dict, None, None
2020-06-01 15:54:49 +00:00
obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
return obj_dict, child_dict, RecurrentTransaction
def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
2020-06-01 15:54:49 +00:00
obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
obj_dict2 = obj_dict.copy()
child_dict2 = child_dict.copy()
child_dict2["membership_id"] = pk_membership
return obj_dict2, child_dict2, MembershipTransaction
def _special_transaction(self, row, obj_dict, child_dict):
# Some transaction uses BDE (idbde=0) as source or destination,
# lets fix that.
2020-06-01 15:54:49 +00:00
obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
field_id = "source_id" if row["type"] == "crédit" else "destination_id"
if "espèce" in row["description"]:
obj_dict[field_id] = 1
elif "carte" in row["description"]:
obj_dict[field_id] = 2
elif "cheques" in row["description"]:
obj_dict[field_id] = 3
elif "virement" in row["description"]:
obj_dict[field_id] = 4
# humans and clubs have always the biggest id
actor_pk = max(row["destinataire"], row["emetteur"])
actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
# custom fields of SpecialTransaction
if actor.__class__.__name__ == "NoteUser":
child_dict["first_name"] = actor.user.first_name
child_dict["last_name"] = actor.user.last_name
else:
child_dict["first_name"] = actor.club.name
child_dict["last_name"] = actor.club.name
return obj_dict, child_dict, SpecialTransaction
def _guest_transaction(self, row, obj_dict, child_dict):
2020-06-01 15:54:49 +00:00
obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
if m:
2020-07-23 05:21:29 +00:00
activity_name = m.group(1)
first_name, last_name = m.group(2), m.group(3)
2020-07-21 23:28:28 +00:00
if first_name == "Marion" and last_name == "Bizu Pose":
first_name, last_name = "Marion Bizu", "Pose"
2020-07-23 05:21:29 +00:00
entry_id = Entry.objects.filter(
activity__name__iexact=activity_name,
guest__first_name__iexact=first_name,
guest__last_name__iexact=last_name,
).first().pk
child_dict["entry_id"] = entry_id
else:
2020-07-25 23:01:17 +00:00
raise Exception(f"Guest not Found {row['id']} first_name, last_name")
return obj_dict, child_dict, GuestTransaction
@timed
@transaction.atomic
2020-07-26 21:59:04 +00:00
def import_transaction(self, cur, chunk_size, idmin, save=True):
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute(
2020-07-25 23:01:17 +00:00
f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
t.destinataire,t.quantite, t.montant, t.description,\
t.valide, t.cantinvalidate, t.categorie, \
a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
FROM transactions AS t \
LEFT JOIN adhesions AS a ON t.id = a.idtransaction \
WHERE t.id >= {idmin} \
ORDER BY t.id;")
n = cur.rowcount
pk_membership = 1
pk_transaction = 1
2020-08-01 13:16:36 +00:00
kfet_balance = 0
for idx, row in enumerate(cur):
2020-07-26 21:59:04 +00:00
if save or idx % chunk_size == 0:
self.update_line(idx, n, row["description"])
2020-07-25 23:01:17 +00:00
MAP_TRANSACTION[row["id"]] = pk_transaction
2020-07-26 21:59:04 +00:00
if not save:
pk_transaction += 1
if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
if not isinstance(note, NoteClub):
pk_transaction += 1
continue
try:
date = make_aware(row["transac_date"])
except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))
# standart transaction object
obj_dict = {
"pk": pk_transaction,
"destination_id": self.MAP_IDBDE[row["destinataire"]],
2020-06-01 15:54:49 +00:00
"polymorphic_ctype": None,
"source_id": self.MAP_IDBDE[row["emetteur"]],
"amount": row["montant"],
2020-06-01 15:54:49 +00:00
"created_at": date,
"destination_alias": "",
"invalidity_reason": "",
"quantity": row["quantite"],
"reason": row["description"],
2020-06-01 15:54:49 +00:00
"source_alias": "",
"valid": row["valide"],
}
2020-07-23 05:21:29 +00:00
if len(obj_dict["reason"]) > 255:
obj_dict["reason"] = obj_dict["reason"][:252] + "..."
# for child transaction Models
2020-06-01 15:54:49 +00:00
child_dict = {"pk": pk_transaction}
ttype = row["type"]
2020-06-01 15:54:49 +00:00
# Membership transaction detection and import
if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=obj_dict["source_id"])
if isinstance(note, NoteClub):
2020-07-26 21:59:04 +00:00
child_transaction = None # don't bother register clubs
else:
user_id = note.user_id
montant = obj_dict["amount"]
2020-06-01 15:54:49 +00:00
(obj_dict0,
child_dict0,
child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
obj_dict0["destination_id"] = 6 # Kfet note id
bde_dict = {
"pk": pk_membership,
"user_id": user_id,
2020-06-01 15:54:49 +00:00
"club_id": BDE_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": min(500, montant)
}
pk_membership += 1
pk_transaction += 1
2020-07-25 23:01:17 +00:00
obj_dict, child_dict, child_transaction =\
self._membership_transaction(row, obj_dict, child_dict, pk_membership)
# Kfet membership
# BDE Membership
obj_dict["pk"] = pk_transaction
child_dict["pk"] = pk_transaction
kfet_dict = {
"pk": pk_membership,
"user_id": user_id,
2020-06-01 15:54:49 +00:00
"club_id": KFET_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": max(montant - 500, 0),
}
obj_dict0["amount"] = bde_dict["fee"]
obj_dict["amount"] = kfet_dict["fee"]
2020-08-01 13:16:36 +00:00
kfet_balance += kfet_dict["fee"]
# BDE membership Transaction is inserted before the Kfet membershipTransaction
pk_membership += 1
pk_transaction += 1
bulk_mgr.add(
2020-06-01 15:54:49 +00:00
Membership(**bde_dict),
Membership(**kfet_dict),
Transaction(**obj_dict0),
child_transaction(**child_dict0),
Transaction(**obj_dict),
child_transaction(**child_dict),
)
continue
elif ttype == "bouton":
obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
elif ttype == "crédit" or ttype == "retrait":
obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
elif ttype == "invitation":
obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
2020-06-01 15:54:49 +00:00
elif ttype == "don" or ttype == "transfert":
obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
else:
child_transaction = None
# create base transaction object and typed one
bulk_mgr.add(Transaction(**obj_dict))
if child_transaction is not None:
2020-07-23 05:21:29 +00:00
child_dict.update(obj_dict)
bulk_mgr.add(child_transaction(**child_dict))
pk_transaction += 1
bulk_mgr.done()
2020-07-21 23:28:28 +00:00
2020-08-01 13:16:36 +00:00
# Update the balance of the BDE and the Kfet club
note_bde = NoteClub.objects.get(pk=5)
note_kfet = NoteClub.objects.get(pk=6)
note_bde.balance -= kfet_balance
note_kfet.balance += kfet_balance
note_bde.save()
note_kfet.save()
2020-07-26 21:59:04 +00:00
@timed
def set_roles(self):
bulk_mgr = BulkCreateManager(chunk_size=10000)
bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
n = len(bde_membership_ids)
for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
self.update_line(idx, n, str(idx))
bulk_mgr.add(
Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
)
bulk_mgr.done()
# Note account has a different treatment
for m in Membership.objects.filter(user_username="note").all():
m.date_end = "3142-12-12"
m.roles.set([20]) # PC Kfet role
m.save()
2020-07-25 23:01:17 +00:00
@timed
@transaction.atomic
def import_remittances(self, cur, chunk_size):
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
2020-07-25 23:01:17 +00:00
n = cur.rowcount
pk_remittance = 1
for idx, row in enumerate(cur):
self.update_line(idx, n, row["commentaire"])
MAP_REMITTANCE[row["id"]] = pk_remittance
remittance_dict = {
"pk": pk_remittance,
2020-07-26 21:59:04 +00:00
"date": make_aware(row["date"]),
2020-07-25 23:01:17 +00:00
"remittance_type_id": 1, # Only Bank checks are supported in NK15
"comment": row["commentaire"],
"closed": row["close"],
}
bulk_mgr.add(Remittance(**remittance_dict))
pk_remittance += 1
bulk_mgr.done()
@timed
def import_checks(self, cur):
cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
"FROM cheques ORDER BY id;")
2020-07-25 23:01:17 +00:00
n = cur.rowcount
for idx, row in enumerate(cur):
self.update_line(idx, n, row["nom"])
2020-07-26 10:05:26 +00:00
if not row["idremise"]:
continue
2020-07-26 21:59:04 +00:00
tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
2020-07-25 23:01:17 +00:00
tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
tr.save()
tr = tr.transaction
tr.last_name = row["nom"]
tr.first_name = row["prenom"]
tr.bank = row["banque"]
2020-07-26 21:59:04 +00:00
try:
tr.save()
except:
print("Failed to save row: " + str(row))
2020-07-21 23:28:28 +00:00
2020-08-07 11:17:17 +00:00
@timed
def import_soge_credits(self):
users = User.objects.filter(profile__registration_valid=True).order_by('pk')
n = users.count()
for idx, user in enumerate(users.all()):
self.update_line(idx, n, user.username)
soge_credit_transaction = SpecialTransaction.objects.filter(
reason__icontains="crédit sogé",
destination_id=user.note.id,
)
if soge_credit_transaction.exists():
soge_credit_transaction = soge_credit_transaction.get()
soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
memberships = Membership.objects.filter(
user=user,
club_id__in=[BDE_PK, KFET_PK],
date_start__lte=soge_credit_transaction.created_at,
2020-08-07 12:00:50 +00:00
date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
2020-08-07 11:17:17 +00:00
).all()
for membership in memberships:
soge_credit.transactions.add(membership.transaction)
soge_credit.save()
@timed
def handle(self, *args, **kwargs):
# default args, provided by ImportCommand.
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
# connecting to nk15 database
conn = pg.connect(database=nk15db, user=nk15user)
cur = conn.cursor(cursor_factory=pge.DictCursor)
if kwargs["map"]:
self.load_map(kwargs["map"])
2020-07-21 23:28:28 +00:00
self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
2020-07-26 21:59:04 +00:00
self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
if not kwargs["nosave"]:
self.set_roles()
2020-07-25 23:01:17 +00:00
self.import_remittances(cur, kwargs["chunk"])
self.import_checks(cur)
2020-08-07 11:17:17 +00:00
self.import_soge_credits()