nk20-scripts/management/commands/import_transaction.py

411 lines
17 KiB
Python

# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
# SPDX-License-Identifier: GPL-3.0-or-later
import copy
import datetime
import re
import pytz
import psycopg2 as pg
import psycopg2.extras as pge
from activity.models import Entry, GuestTransaction
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.db import transaction
from django.utils.timezone import make_aware
from member.models import Membership
from note.models import (MembershipTransaction, Note, NoteClub,
RecurrentTransaction, SpecialTransaction,
TemplateCategory, Transaction, TransactionTemplate)
from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy
from ._import_utils import BulkCreateManager, ImportCommand, timed
MAP_TRANSACTION = dict()
MAP_REMITTANCE = dict()
# from member/fixtures/initial
BDE_PK = 1
KFET_PK = 2
# from note/fixtures/initial
NOTE_SPECIAL_CODE = {
"espèce": 1,
"carte": 2,
"chèque": 3,
"virement": 4,
}
# from permission/fixtures/initial
BDE_ROLE_PK = 1
KFET_ROLE_PK = 2
CT = {
"RecurrentTransaction": ContentType.objects.get(app_label="note", model="recurrenttransaction"),
"SpecialTransaction": ContentType.objects.get(app_label="note", model="specialtransaction"),
"MembershipTransaction": ContentType.objects.get(app_label="note", model="membershiptransaction"),
"GuestTransaction": ContentType.objects.get(app_label="activity", model="guesttransaction"),
}
def get_date_end(date_start):
date_end = copy.deepcopy(date_start)
if date_start.month >= 8:
date_end = date_start.replace(year=date_start.year + 1)
date_end = date_end.replace(month=9, day=30)
return date_end
class Command(ImportCommand):
"""
Import command for People base data (Comptes, and Aliases)
"""
def add_arguments(self, parser):
parser.add_argument('-b', '--buttons', action='store_true', help="import buttons")
parser.add_argument('-t', '--transactions', action='store', default=0, help="start id for transaction import")
parser.add_argument('-n', '--nosave', action='store_true', default=False, help="Scan only transactions, "
"don't save them")
@timed
def import_buttons(self, cur, chunk_size, import_buttons):
self.categories = dict()
self.buttons = dict()
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute("SELECT * FROM boutons;")
n = cur.rowcount
for idx, row in enumerate(cur):
self.update_line(idx, n, row["label"])
if row["categorie"] not in self.categories:
cat = TemplateCategory.objects.get_or_create(name=row["categorie"])[0]
cat.save()
self.categories[row["categorie"]] = cat.pk
obj_dict = {
"pk": row["id"],
"name": row["label"],
"amount": row["montant"],
"destination_id": self.MAP_IDBDE[row["destinataire"]],
"category_id": self.categories[row["categorie"]],
"display": row["affiche"],
"description": row["description"],
}
if row["label"] in self.buttons:
obj_dict["name"] = f"{obj_dict['name']}_{obj_dict['destination_id']}"
if import_buttons:
bulk_mgr.add(TransactionTemplate(**obj_dict))
self.buttons[obj_dict["name"]] = (row["id"], self.categories[row["categorie"]])
bulk_mgr.done()
def _basic_transaction(self, row, obj_dict, child_dict):
if len(row["description"]) > 255:
obj_dict["reason"] = obj_dict["reason"][:252] + "…)"
return obj_dict, None, None
def _template_transaction(self, row, obj_dict, child_dict):
if self.buttons.get(row["description"]):
child_dict["template_id"] = self.buttons[row["description"]][0]
# elif self.categories.get(row["categorie"]):
# child_dict["category_id"] = self.categories[row["categorie"]]
elif "WEI" in row["description"]:
return obj_dict, None, None
else:
return obj_dict, None, None
obj_dict["polymorphic_ctype"] = CT["RecurrentTransaction"]
return obj_dict, child_dict, RecurrentTransaction
def _membership_transaction(self, row, obj_dict, child_dict, pk_membership):
obj_dict["polymorphic_ctype"] = CT["MembershipTransaction"]
obj_dict2 = obj_dict.copy()
child_dict2 = child_dict.copy()
child_dict2["membership_id"] = pk_membership
return obj_dict2, child_dict2, MembershipTransaction
def _special_transaction(self, row, obj_dict, child_dict):
# Some transaction uses BDE (idbde=0) as source or destination,
# lets fix that.
obj_dict["polymorphic_ctype"] = CT["SpecialTransaction"]
field_id = "source_id" if row["type"] == "crédit" else "destination_id"
if "espèce" in row["description"]:
obj_dict[field_id] = 1
elif "carte" in row["description"]:
obj_dict[field_id] = 2
elif "cheques" in row["description"]:
obj_dict[field_id] = 3
elif "virement" in row["description"]:
obj_dict[field_id] = 4
# humans and clubs have always the biggest id
actor_pk = max(row["destinataire"], row["emetteur"])
actor = Note.objects.get(id=self.MAP_IDBDE[actor_pk])
# custom fields of SpecialTransaction
if actor.__class__.__name__ == "NoteUser":
child_dict["first_name"] = actor.user.first_name
child_dict["last_name"] = actor.user.last_name
else:
child_dict["first_name"] = actor.club.name
child_dict["last_name"] = actor.club.name
return obj_dict, child_dict, SpecialTransaction
def _guest_transaction(self, row, obj_dict, child_dict):
obj_dict["polymorphic_ctype"] = CT["GuestTransaction"]
m = re.search(r"Invitation (.*?)(?:\s\()(.*?)\s(.*?)\)", row["description"])
if m:
activity_name = m.group(1)
first_name, last_name = m.group(2), m.group(3)
if first_name == "Marion" and last_name == "Bizu Pose":
first_name, last_name = "Marion Bizu", "Pose"
entry_id = Entry.objects.filter(
activity__name__iexact=activity_name,
guest__first_name__iexact=first_name,
guest__last_name__iexact=last_name,
).first().pk
child_dict["entry_id"] = entry_id
else:
raise Exception(f"Guest not Found {row['id']} first_name, last_name")
return obj_dict, child_dict, GuestTransaction
@timed
@transaction.atomic
def import_transaction(self, cur, chunk_size, idmin, save=True):
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute(
f"SELECT t.id, t.date AS transac_date, t.type, t.emetteur,\
t.destinataire,t.quantite, t.montant, t.description,\
t.valide, t.cantinvalidate, t.categorie, \
a.idbde, a.annee, a.wei, a.date AS adh_date, a.section\
FROM transactions AS t \
LEFT JOIN adhesions AS a ON t.id = a.idtransaction \
WHERE t.id >= {idmin} \
ORDER BY t.id;")
n = cur.rowcount
pk_membership = 1
pk_transaction = 1
kfet_balance = 0
for idx, row in enumerate(cur):
if save or idx % chunk_size == 0:
self.update_line(idx, n, row["description"])
MAP_TRANSACTION[row["id"]] = pk_transaction
if not save:
pk_transaction += 1
if row["valide"] and (row["type"] == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=self.MAP_IDBDE[row["emetteur"]])
if not isinstance(note, NoteClub):
pk_transaction += 1
continue
try:
date = make_aware(row["transac_date"])
except (pytz.NonExistentTimeError, pytz.AmbiguousTimeError):
date = make_aware(row["transac_date"] + datetime.timedelta(hours=1))
# standart transaction object
obj_dict = {
"pk": pk_transaction,
"destination_id": self.MAP_IDBDE[row["destinataire"]],
"polymorphic_ctype": None,
"source_id": self.MAP_IDBDE[row["emetteur"]],
"amount": row["montant"],
"created_at": date,
"destination_alias": "",
"invalidity_reason": "",
"quantity": row["quantite"],
"reason": row["description"],
"source_alias": "",
"valid": row["valide"],
}
if len(obj_dict["reason"]) > 255:
obj_dict["reason"] = obj_dict["reason"][:254] + ""
# for child transaction Models
child_dict = {"pk": pk_transaction}
ttype = row["type"]
# Membership transaction detection and import
if row["valide"] and (ttype == "adhésion" or row["description"].lower() == "inscription"):
note = Note.objects.get(pk=obj_dict["source_id"])
if isinstance(note, NoteClub):
child_transaction = None # don't bother register clubs
else:
user_id = note.user_id
montant = obj_dict["amount"]
(obj_dict0,
child_dict0,
child_transaction) = self._membership_transaction(row, obj_dict, child_dict, pk_membership)
obj_dict0["destination_id"] = 6 # Kfet note id
bde_dict = {
"pk": pk_membership,
"user_id": user_id,
"club_id": BDE_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": min(500, montant)
}
pk_membership += 1
pk_transaction += 1
obj_dict, child_dict, child_transaction =\
self._membership_transaction(row, obj_dict, child_dict, pk_membership)
# Kfet membership
# BDE Membership
obj_dict["pk"] = pk_transaction
child_dict["pk"] = pk_transaction
kfet_dict = {
"pk": pk_membership,
"user_id": user_id,
"club_id": KFET_PK,
"date_start": date.date(), # Only date, not time
"date_end": get_date_end(date.date()),
"fee": max(montant - 500, 0),
}
obj_dict0["amount"] = bde_dict["fee"]
obj_dict["amount"] = kfet_dict["fee"]
kfet_balance += kfet_dict["fee"]
# BDE membership Transaction is inserted before the Kfet membershipTransaction
pk_membership += 1
pk_transaction += 1
bulk_mgr.add(
Membership(**bde_dict),
Membership(**kfet_dict),
Transaction(**obj_dict0),
child_transaction(**child_dict0),
Transaction(**obj_dict),
child_transaction(**child_dict),
)
continue
elif ttype == "bouton":
obj_dict, child_dict, child_transaction = self._template_transaction(row, obj_dict, child_dict)
elif ttype == "crédit" or ttype == "retrait":
obj_dict, child_dict, child_transaction = self._special_transaction(row, obj_dict, child_dict)
elif ttype == "invitation":
obj_dict, child_dict, child_transaction = self._guest_transaction(row, obj_dict, child_dict)
elif ttype == "don" or ttype == "transfert":
obj_dict, child_dict, child_transaction = self._basic_transaction(row, obj_dict, child_dict)
else:
child_transaction = None
# create base transaction object and typed one
bulk_mgr.add(Transaction(**obj_dict))
if child_transaction is not None:
child_dict.update(obj_dict)
bulk_mgr.add(child_transaction(**child_dict))
pk_transaction += 1
bulk_mgr.done()
# Update the balance of the BDE and the Kfet club
note_bde = NoteClub.objects.get(pk=5)
note_kfet = NoteClub.objects.get(pk=6)
note_bde.balance -= kfet_balance
note_kfet.balance += kfet_balance
note_bde.save()
note_kfet.save()
@timed
def set_roles(self):
bulk_mgr = BulkCreateManager(chunk_size=10000)
bde_membership_ids = Membership.objects.filter(club__pk=BDE_PK).values_list('id', flat=True)
kfet_membership_ids = Membership.objects.filter(club__pk=KFET_PK).values_list('id', flat=True)
n = len(bde_membership_ids)
for idx, (m_bde_id, m_kfet_id) in enumerate(zip(bde_membership_ids, kfet_membership_ids)):
self.update_line(idx, n, str(idx))
bulk_mgr.add(
Membership.roles.through(membership_id=m_bde_id, role_id=BDE_ROLE_PK),
Membership.roles.through(membership_id=m_kfet_id, role_id=KFET_ROLE_PK),
)
bulk_mgr.done()
# Note account has a different treatment
for m in Membership.objects.filter(user_username="note").all():
m.date_end = "3142-12-12"
m.roles.set([20]) # PC Kfet role
m.save()
@timed
@transaction.atomic
def import_remittances(self, cur, chunk_size):
bulk_mgr = BulkCreateManager(chunk_size=chunk_size)
cur.execute("SELECT id, date, commentaire, close FROM remises ORDER BY id;")
n = cur.rowcount
pk_remittance = 1
for idx, row in enumerate(cur):
self.update_line(idx, n, row["commentaire"])
MAP_REMITTANCE[row["id"]] = pk_remittance
remittance_dict = {
"pk": pk_remittance,
"date": make_aware(row["date"]),
"remittance_type_id": 1, # Only Bank checks are supported in NK15
"comment": row["commentaire"],
"closed": row["close"],
}
bulk_mgr.add(Remittance(**remittance_dict))
pk_remittance += 1
bulk_mgr.done()
@timed
def import_checks(self, cur):
cur.execute("SELECT id, nom, prenom, banque, idtransaction, idremise "
"FROM cheques ORDER BY id;")
n = cur.rowcount
for idx, row in enumerate(cur):
self.update_line(idx, n, row["nom"])
if not row["idremise"]:
continue
tr = SpecialTransactionProxy.objects.get_or_create(transaction_id=MAP_TRANSACTION[row["idtransaction"]])[0]
tr.remittance_id = MAP_REMITTANCE[row["idremise"]]
tr.save()
tr = tr.transaction
tr.last_name = row["nom"]
tr.first_name = row["prenom"]
tr.bank = row["banque"]
try:
tr.save()
except:
print("Failed to save row: " + str(row))
@timed
def import_soge_credits(self):
users = User.objects.filter(profile__registration_valid=True).order_by('pk')
n = users.count()
for idx, user in enumerate(users.all()):
self.update_line(idx, n, user.username)
soge_credit_transaction = SpecialTransaction.objects.filter(
reason__icontains="crédit sogé",
destination_id=user.note.id,
)
if soge_credit_transaction.exists():
soge_credit_transaction = soge_credit_transaction.get()
soge_credit = SogeCredit.objects.create(user=user, credit_transaction=soge_credit_transaction)
memberships = Membership.objects.filter(
user=user,
club_id__in=[BDE_PK, KFET_PK],
date_start__lte=soge_credit_transaction.created_at,
date_end__gte=soge_credit_transaction.created_at + datetime.timedelta(days=61),
).all()
for membership in memberships:
soge_credit.transactions.add(membership.transaction)
soge_credit.save()
@timed
def handle(self, *args, **kwargs):
# default args, provided by ImportCommand.
nk15db, nk15user = kwargs['nk15db'], kwargs['nk15user']
# connecting to nk15 database
conn = pg.connect(database=nk15db, user=nk15user)
cur = conn.cursor(cursor_factory=pge.DictCursor)
if kwargs["map"]:
self.load_map(kwargs["map"])
self.import_buttons(cur, kwargs["chunk"], kwargs["buttons"])
self.import_transaction(cur, kwargs["chunk"], kwargs["transactions"], not kwargs["nosave"])
if not kwargs["nosave"]:
self.set_roles()
self.import_remittances(cur, kwargs["chunk"])
self.import_checks(cur)
self.import_soge_credits()