borg backup

Remove useless tee, use "mkdir -p" and use .sql.gz.

.gitignore

@@ -33,6 +33,7 @@ coverage
 
 # Local data
 secrets.py
+*/.env_borg
 *.log
 
 # Virtualenv

README.md

@@ -1,63 +1,3 @@
 # Script de la NoteKfet 2020
 
-## Commandes Django
+La documentation est disponible sur <https://note.crans.org/doc/scripts/>.
-
-> les commandes sont documentées:
-> `./manage.py command --help`
-
-- `import_nk15` :
-
-    Importe un dump de la NoteKfet 2015. 
-
-- `make_su [--STAFF|-s] [--SUPER|-S]` :
-
-    Rend actifs les pseudos renseignés.
-  
-    * Si `--STAFF` ou `-s` est renseigné, donne en plus le statut d'équipe aux pseudos renseignés, 
-    permettant l'accès à l'interface admin.
-    * Si `--SUPER` ou `-S` est renseigné, donne en plus le statut de super-utilisateur aux pseudos renseignés,
-    octroyant tous les droits sur la plateforme.
-
-- `wei_algorithm` :
-
-    Lance l'algorithme de répartition des 1A au dernier WEI. Cela a pour effet de suggérer un bus pour tous les 1A
-    inscrits au dernier WEI en fonction des données rentrées dans le sondage, la validation se faisant ensuite
-    manuellement via l'interface Web.
-
-- `extract_ml_registrations --type {members, clubs, events, art, sport} [--year|-y YEAR]` :
-
-    Récupère la liste des adresses mail à inscrire à une liste de diffusion donnée.
-    
-    * `members` : Liste des adresses mail des utilisateurs ayant une adhésion BDE (et non Kfet) active.
-    * `clubs` : Liste des adresses mail de contact de tous les clubs BDE enregistrés.
-    * `events` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                 la liste de diffusion des événements du BDE.
-    * `art` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-              la liste de diffusion concertnant les actualités artistiques du BDA.
-    * `sport` : Liste de toutes les adresses mails des utilisateurs inscrits au WEI ayant demandé à s'inscrire sur
-                la liste de diffusion concertnant les actualités sportives du BDS.
-    
-    Le champ `--year` est optionnel : il permet de choisir l'année du WEI en question (pour les trois dernières 
-    options). Si non renseigné, il s'agit du dernier WEI.
-    
-    Par défaut, si `--type` est non renseigné, la liste des adhérents BDE est renvoyée.
-
-- `extract_wei_registrations [--year|-y YEAR] [--bus|-b BUS] [--team|-t TEAM] [--sep SEP]` :
-
-    Récupère la liste des inscriptions au WEI et l'exporte au format CSV. Arguments possibles, optionnels :
-    
-    * `--year YEAR` : sélectionne l'année du WEI. Par défaut, il s'agit du dernier WEI ayant eu lieu.
-    * `--bus BUS` : filtre par bus, en récupérant uniquement les inscriptions sur un bus. Par défaut, on affiche
-    tous les bus.
-    * `--team TEAM` : filtre par équipe, en récupérant uniquement les inscriptions sur une équipe. Par défaut, on
-    affiche toutes les équipes. Entrer `"none"` filtre les inscriptions sans équipe (chefs de bus, ...)
-    * `--sep` : définit le caractère de séparation des colonnes du fichier CSV. Par défaut, il s'agit du caractère `|`.
-    Merci de ne pas rentrer plus d'un caractère.
-    
-
-## Shell
-
-- Tabula rasa :
-```shell script
-sudo -u postgres sh -c "dropdb note_db && psql -c 'CREATE DATABASE note_db OWNER note;'"
-```

__init__.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 default_app_config = 'scripts.apps.ScriptsConfig'

apps.py

@@ -1,8 +1,7 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.apps import AppConfig
-from django.core.signals import got_request_exception
 
 
 class ScriptsConfig(AppConfig):

management/commands/_import_utils.py

@@ -1,14 +1,13 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import json
 import time
 from collections import defaultdict
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import transaction
-
 from polymorphic.models import PolymorphicModel
 
 
@@ -16,6 +15,7 @@ def timed(method):
     """"
     A simple decorator to measure time elapsed in class function (hence the args[0])
     """
+
     def _timed(*args, **kw):
         ts = time.time()
         result = method(*args, **kw)

management/commands/anonymize_data.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2023 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.core.management.base import BaseCommand
@@ -7,11 +7,39 @@ from django.db import connection
 
 class Command(BaseCommand):
     """
-    Command to protect sensitive data during the beta phase, to prevent a right escalation.
+    Command to protect sensitive data during the beta phase or after WEI.
-    Phone number, email address, postal address, first and last name are removed.
+    Phone number, email address, postal address, first and last name,
+    IP addresses, health issues, gender and birth date are removed.
     """
-    def handle(self, *args, **kwargs):
+    def add_arguments(self, parser):
+        parser.add_argument('--force', '-f', action='store_true', help="Actually anonymize data.")
+        parser.add_argument('--type', '-t', choices=["all", "wei", "user"], default="",
+                            help='Select the type of data to anonyze (default None)')
+
+    def handle(self, *args, **options):
+        if not options['force']:
+            if options['type'] == "all":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset ALL personal data with "
+                              "sample data. Don't use  in production! If you know what you are doing, please "
+                              "add --force option.")
+            elif options['type'] == "wei":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset WEI personal data with "
+                              "sample data. Use it in production only after WEI. If you know what you are doing,"
+                              "please add --force option.")
+            elif options['type'] == "user":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data "
+                              "visible by user (not admin or trez BDE) with sample data. Don't use  in "
+                              "production! If you know what you are doing, please add --force option.")
+            else:
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data with "
+                              "sample data. Don't use  in production ('wei' can be use in production after "
+                              "the WEI)! If you know what you are doing, please choose a type.")
+            exit(1)
+
         cur = connection.cursor()
+        if options['type'] in ("all","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize profile, user club and guest data")
             cur.execute("UPDATE member_profile SET "
                     "phone_number = '0123456789', "
                     "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
@@ -21,4 +49,45 @@ class Command(BaseCommand):
                     "email = 'anonymous@example.com';")
             cur.execute("UPDATE member_club SET "
                     "email = 'anonymous@example.com';")
+            cur.execute("UPDATE activity_guest SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+
+        if options['type'] in ("all","wei","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize WEI data")
+            cur.execute("UPDATE wei_weiregistration SET "
+                    "birth_date = '1998-01-08', "
+                    "emergency_contact_name = 'Anne Onyme', "
+                    "emergency_contact_phone = '0123456789', "
+                    "gender = 'nonbinary', "
+                    "health_issues = 'Tout va bien';")
+
+        if options['type'] == "all":
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize invoice, special transaction, log, mailer and oauth data")
+            cur.execute("UPDATE treasury_invoice SET "
+                    "name = 'Anne Onyme', "
+                    "object = 'Rends nous riches', "
+                    "description = 'Donne nous plein de sous', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE treasury_product SET "
+                    "designation = 'un truc inutile';")
+            cur.execute("UPDATE note_specialtransaction SET "
+                    "bank = 'le matelas', "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+            cur.execute("UPDATE logs_changelog SET "
+                    "ip = '127.0.0.1', "
+                    "data = 'new data', "
+                    "previous = 'old data';")
+            cur.execute("UPDATE mailer_messagelog SET "
+                    "log_message = 'log message', "
+                    "message_data = 'message data';")
+            cur.execute("UPDATE mailer_dontsendentry SET "
+                    "to_address = 'anonymous@example.com';")
+            cur.execute("UPDATE oauth2_provider_application SET "
+                    "name = 'external app', "
+                    "redirect_uris = 'http://external.app', "
+                    "client_secret = 'abcdefghijklmnopqrstuvwxyz';")
         cur.close()

management/commands/check_consistency.py

@@ -1,6 +1,6 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
-from django.core.mail import send_mail
+
 from django.core.management import BaseCommand
 from django.db.models import Sum, F
 
@@ -14,7 +14,6 @@ class Command(BaseCommand):
         parser.add_argument('--check-all', '-a', action='store_true', help='Check all notes')
         parser.add_argument('--check', '-c', type=int, nargs='+', help='Select note ids')
         parser.add_argument('--fix', '-f', action='store_true', help='Fix note balances')
-        parser.add_argument('--mail', '-m', action='store_true', help='Send mail to admins if there is an error')
 
     def handle(self, *args, **options):
         error = False
@@ -23,9 +22,10 @@ class Command(BaseCommand):
         if options["sum_all"]:
             s = Note.objects.aggregate(Sum("balance"))["balance__sum"]
             if s:
-                err_log += self.style.NOTICE("LA SOMME DES NOTES NE VAUT PAS ZÉRO : " + pretty_money(s)) + "\n"
+                self.stderr.write(self.style.NOTICE("LA SOMME DES NOTES NE VAUT PAS ZÉRO : " + pretty_money(s)))
                 error = True
             else:
+                if options["verbosity"] > 0:
                     self.stdout.write(self.style.SUCCESS("La somme des notes vaut bien zéro."))
 
         notes = Note.objects.none()
@@ -42,19 +42,13 @@ class Command(BaseCommand):
                 .annotate(total=F("quantity") * F("amount")).aggregate(Sum("total"))["total__sum"] or 0
             calculated_balance = incoming - outcoming
             if calculated_balance != balance:
-                err_log += self.style.NOTICE("LA SOMME DES TRANSACTIONS DE LA NOTE {} NE CORRESPOND PAS "
+                self.stderr.write(self.style.NOTICE(f"LA SOMME DES TRANSACTIONS DE LA NOTE {note} NE CORRESPOND PAS "
-                                             "AVEC LE MONTANT RÉEL".format(str(note))) + "\n"
+                                                    "AVEC LE MONTANT RÉEL"))
-                err_log += self.style.NOTICE("Attendu : {}, calculé : {}"
+                self.stderr.write(self.style.NOTICE(f"Attendu : {pretty_money(balance)}, "
-                                             .format(pretty_money(balance), pretty_money(calculated_balance))) + "\n"
+                                                    f"calculé : {pretty_money(calculated_balance)}"))
                 if options["fix"]:
                     note.balance = calculated_balance
                     note.save()
                 error = True
 
-        if error:
-            self.stderr.write(err_log)
-            if options["mail"]:
-                send_mail("[Note Kfet] La base de données n'est pas consistante", err_log,
-                          "NoteKfet2020 <notekfet2020@crans.org>", ["respo-info.bde@lists.crans.org"])
-
         exit(1 if error else 0)

management/commands/compilejsmessages.py

@@ -1,5 +1,6 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
+
 import os
 
 from django.conf import settings
@@ -12,17 +13,16 @@ class Command(BaseCommand):
     """
     Generate Javascript translation files
     """
-    def add_arguments(self, parser):
-        parser.add_argument('--out', '-o', type=str, default='static', help='Output directory, where static files are.')
 
     def handle(self, *args, **kwargs):
         for code, _ in settings.LANGUAGES:
             if code == settings.LANGUAGE_CODE:
                 continue
+            if kwargs["verbosity"] > 0:
                 self.stdout.write(f"Generate {code} javascript localization file")
             with translation.override(code):
                 resp = JavaScriptCatalog().get(None, packages="member+note")
-                if not os.path.isdir(kwargs["out"] + "/js/jsi18n"):
+                if not os.path.isdir(settings.STATIC_ROOT + "/js/jsi18n"):
-                    os.makedirs(kwargs["out"] + "/js/jsi18n")
+                    os.makedirs(settings.STATIC_ROOT + "/js/jsi18n")
-                with open(kwargs["out"] + f"/js/jsi18n/{code}.js", "wb") as f:
+                with open(settings.STATIC_ROOT + f"/js/jsi18n/{code}.js", "wb") as f:
                     f.write(resp.content)

management/commands/extract_ml_registrations.py

@@ -1,18 +1,16 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2024 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from datetime import date
 
 from django.contrib.auth.models import User
 from django.core.management import BaseCommand
-from django.db.models import Q
+from member.models import Club, Membership
-from member.models import Membership, Club
-from wei.models import WEIClub
 
 
 class Command(BaseCommand):
     help = "Get mailing list registrations from the last wei. " \
-           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -t {fr, en}. " \
+           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -l {fr, en} -y {0, 1, ...}. " \
            "You can write this into a file with a pipe, then paste the document into your mail manager."
 
     def add_arguments(self, parser):
@@ -21,15 +19,23 @@ class Command(BaseCommand):
         parser.add_argument('--lang', '-l', type=str, choices=['fr', 'en'], default='fr',
                             help='Select the registred users of the ML of the given language. Useful only for the '
                                  'events mailing list.')
+        parser.add_argument('--years', '-y', type=int, default=0, 
+                            help='Select the cumulative registred users of a membership from years ago. 0 means the current users')
 
     def handle(self, *args, **options):
         # TODO: Improve the mailing list extraction system, and link it automatically with Mailman.
 
+        if options['verbosity'] == 0:
+            # This is useless, but this what the user asked.
+            return
+
         if options["type"] == "members":
+            today_date = date.today()
+            selected_date = date(today_date.year - options["years"], today_date.month, today_date.day)
             for membership in Membership.objects.filter(
                 club__name="BDE",
-                date_start__lte=date.today(),
+                date_start__lte=today_date,
-                date_end__gte=date.today(),
+                date_end__gte=selected_date,
             ).all():
                 self.stdout.write(membership.user.email)
             return
@@ -47,8 +53,11 @@ class Command(BaseCommand):
             return
 
         if options["type"] == "art":
+            nb=0
             for user in User.objects.filter(profile__ml_art_registration=True).all():
                 self.stdout.write(user.email)
+                nb+=1
+            self.stdout.write(str(nb))
             return
 
         if options["type"] == "sport":

management/commands/force_delete_user.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import getpass
@@ -11,7 +11,6 @@ from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.db.models import Q
 from django.test import override_settings
-
 from note.models import Alias, Transaction
 
 
@@ -57,13 +56,13 @@ class Command(BaseCommand):
                 if user_id.isnumeric():
                     qs = User.objects.filter(pk=int(user_id))
                     if not qs.exists():
-                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring..."))
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
                         continue
                     user = qs.get()
                 else:
                     qs = Alias.objects.filter(normalized_name=Alias.normalize(user_id), note__noteuser__isnull=False)
                     if not qs.exists():
-                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring..."))
+                        self.stderr.write(self.style.WARNING(f"User {user_id} was not found. Ignoring…"))
                         continue
                     user = qs.get().note.user
 
@@ -78,18 +77,18 @@ class Command(BaseCommand):
                     # Deleting transactions
                     transactions = Transaction.objects.filter(Q(source=user.note) | Q(destination=user.note)).all()
                     local_deleted += list(transactions)
-                    if kwargs['verbosity'] >= 1:
                     for tr in transactions:
-                            self.stdout.write(f"Removing {tr}...")
+                        if kwargs['verbosity'] >= 1:
+                            self.stdout.write(f"Removing {tr}…")
                         if force:
-                        transactions.delete()
+                            tr.delete()
 
                     # Deleting memberships
                     memberships = user.memberships.all()
                     local_deleted += list(memberships)
                     if kwargs['verbosity'] >= 1:
                         for membership in memberships:
-                            self.stdout.write(f"Removing {membership}...")
+                            self.stdout.write(f"Removing {membership}…")
                     if force:
                         memberships.delete()
 
@@ -98,7 +97,7 @@ class Command(BaseCommand):
                     local_deleted += list(alias_set)
                     if kwargs['verbosity'] >= 1:
                         for alias in alias_set:
-                            self.stdout.write(f"Removing alias {alias}...")
+                            self.stdout.write(f"Removing alias {alias}…")
                     if force:
                         alias_set.delete()
 
@@ -110,7 +109,7 @@ class Command(BaseCommand):
                         local_deleted += list(entries)
                         if kwargs['verbosity'] >= 1:
                             for entry in entries:
-                                self.stdout.write(f"Removing {entry}...")
+                                self.stdout.write(f"Removing {entry}…")
                         if force:
                             entries.delete()
 
@@ -119,7 +118,7 @@ class Command(BaseCommand):
                         local_deleted += list(guests)
                         if kwargs['verbosity'] >= 1:
                             for guest in guests:
-                                self.stdout.write(f"Removing guest {guest}...")
+                                self.stdout.write(f"Removing guest {guest}…")
                         if force:
                             guests.delete()
 
@@ -131,7 +130,7 @@ class Command(BaseCommand):
                         local_deleted += list(credits)
                         if kwargs['verbosity'] >= 1:
                             for credit in credits:
-                                self.stdout.write(f"Removing {credit}...")
+                                self.stdout.write(f"Removing {credit}…")
                         if force:
                             credits.delete()
 
@@ -169,8 +168,9 @@ class Command(BaseCommand):
                         deleted += local_deleted
 
         if deleted_users:
-            message = f"Les utilisateurs {deleted_users} ont été supprimés par {executor}.\n\n"
+            message = f"Les utilisateur⋅rices {deleted_users} ont été supprimé⋅es par {executor}.\n\n"
             message += "Ont été supprimés en conséquence les objets suivants :\n\n"
             for obj in deleted:
                 message += f"{repr(obj)} (pk: {obj.pk})\n"
-            mail_admins("Utilisateurs supprimés", message)
+            if force and kwargs['doit']:
+                mail_admins("Utilisateur⋅rices supprimés", message)

management/commands/import_account.py

@@ -1,23 +1,19 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import psycopg2 as pg
-import psycopg2.extras as pge
 import datetime
 import json
 
-from django.template.loader import render_to_string
+import psycopg2 as pg
-from django.utils.timezone import make_aware, now
+import psycopg2.extras as pge
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from note.models import Note, NoteUser, NoteClub
-from note.models import Alias
 from member.models import Club, Profile
+from note.models import Alias, Note, NoteClub, NoteUser
 
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 M_DURATION = 396
 M_START = datetime.date(2019, 8, 1)
@@ -214,7 +210,7 @@ class Command(ImportCommand):
         pk_alias = Alias.objects.order_by('-id').first().id + 1
         for idx, row in enumerate(cur):
             alias_name = row["alias"]
-            alias_name = (alias_name[:252] + '...') if len(alias_name) > 255 else alias_name
+            alias_name = (alias_name[:254] + '…') if len(alias_name) > 255 else alias_name
             alias_norm = Alias.normalize(alias_name)
             self.update_line(idx, n, alias_norm)
             # clean pseudo (normalized pseudo must be unique)

management/commands/import_activities.py

@@ -1,18 +1,15 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import psycopg2 as pg
 import psycopg2.extras as pge
-import datetime
+from activity.models import Activity, ActivityType, Entry, Guest
-import copy
-
-from django.utils.timezone import make_aware
 from django.db import transaction
-
+from django.utils.timezone import make_aware
-from activity.models import ActivityType, Activity, Guest, Entry
 from member.models import Club
 from note.models import Note, NoteUser
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_ACTIVITY = dict()
 
@@ -29,6 +26,7 @@ CLUB_RELOU = [
     2365,  # Pot Vieux
 ]
 
+
 class Command(ImportCommand):
     """
     Import command for Activities Base Data (Comptes, and Aliases)

management/commands/import_nk15.py

@@ -1,13 +1,13 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import subprocess
 
-from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
 from ._import_utils import ImportCommand
 
+
 class Command(ImportCommand):
     """
     Command for importing the database of NK15.

management/commands/import_transaction.py

@@ -1,31 +1,25 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+import copy
+import datetime
 import re
+
+import pytz
 import psycopg2 as pg
 import psycopg2.extras as pge
-import pytz
+from activity.models import Entry, GuestTransaction
-import datetime
-import copy
-
 from django.contrib.auth.models import User
-from django.utils.timezone import make_aware
-from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-
+from django.db import transaction
-from note.models import (TemplateCategory,
+from django.utils.timezone import make_aware
-                         TransactionTemplate,
-                         Transaction,
-                         RecurrentTransaction,
-                         SpecialTransaction,
-                         MembershipTransaction,
-                         )
-from note.models import Note, NoteClub
-from activity.models import Guest, GuestTransaction, Entry
-
 from member.models import Membership
-from treasury.models import Remittance, SpecialTransactionProxy, SogeCredit
+from note.models import (MembershipTransaction, Note, NoteClub,
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+                         RecurrentTransaction, SpecialTransaction,
+                         TemplateCategory, Transaction, TransactionTemplate)
+from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_TRANSACTION = dict()
 MAP_REMITTANCE = dict()
@@ -102,7 +96,7 @@ class Command(ImportCommand):
 
     def _basic_transaction(self, row, obj_dict, child_dict):
         if len(row["description"]) > 255:
-            obj_dict["reason"] = obj_dict["reason"][:250] + "...)"
+            obj_dict["reason"] = obj_dict["reason"][:252] + "…)"
         return obj_dict, None, None
 
     def _template_transaction(self, row, obj_dict, child_dict):
@@ -221,7 +215,7 @@ class Command(ImportCommand):
                 "valid": row["valide"],
             }
             if len(obj_dict["reason"]) > 255:
-                obj_dict["reason"] = obj_dict["reason"][:252] + "..."
+                obj_dict["reason"] = obj_dict["reason"][:254] + "…"
             # for child transaction Models
             child_dict = {"pk": pk_transaction}
             ttype = row["type"]

management/commands/make_su.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.core.management.base import BaseCommand
@@ -16,7 +16,11 @@ class Command(BaseCommand):
             user = User.objects.get(username=uname)
             user.is_active = True
             if kwargs['STAFF']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to staff users…")
                 user.is_staff = True
             if kwargs['SUPER']:
+                if kwargs['verbosity'] > 0:
+                    self.stdout.write(f"Add {user} to superusers…")
                 user.is_superuser = True
             user.save()

management/commands/refresh_activities.py

@@ -1,15 +1,15 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os
-from bs4 import BeautifulSoup
-from django.core.management import BaseCommand
-from django.urls import reverse
-from django.utils import timezone
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
+from bs4 import BeautifulSoup
 from activity.models import Activity
+from django.core.management import BaseCommand
+from django.urls import reverse
+from django.utils import timezone
 
 
 class Command(BaseCommand):
@@ -100,7 +100,7 @@ class Command(BaseCommand):
                 title=act.name,
                 start=timezone.localtime(act.date_start).strftime("%Y-%m-%d %H:%M"),
                 end=timezone.localtime(act.date_end).strftime("%Y-%m-%d %H:%M"),
-                description=act.description.replace("\r", "").replace("\n", "<<BR>>"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
                 club=act.organizer.name,
                 location=act.location,
             )
@@ -108,7 +108,7 @@ class Command(BaseCommand):
             return "|| {start} || {title} || {description} || {club} || {location} ||".format(
                 title=act.name,
                 start=timezone.localtime(act.date_start).strftime("%d/%m/%Y"),
-                description=act.description.replace("\r", "").replace("\n", "<<BR>>"),
+                description=act.description.replace("\r", "").replace("\n", " <<BR>>"),
                 club=act.organizer.name,
                 location=act.location,
             )

management/commands/refresh_highlighted_buttons.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from datetime import timedelta
@@ -6,7 +6,6 @@ from datetime import timedelta
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from django.utils import timezone
-
 from note.models import RecurrentTransaction, TransactionTemplate
 
 
@@ -23,6 +22,7 @@ class Command(BaseCommand):
         for d in queryset.all():
             button_id = d["template"]
             button = TransactionTemplate.objects.get(pk=button_id)
+            if kwargs['verbosity'] > 0:
                 self.stdout.write(self.style.WARNING("Highlight button {name} ({count:d} transactions)..."
                                                      .format(name=button.name, count=d["transaction_count"])))
             button.highlighted = True

management/commands/send_mail_to_negative_balances.py

@@ -1,15 +1,14 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from datetime import date
+from datetime import date, timedelta
 
 from django.core.mail import send_mail
 from django.core.management import BaseCommand
 from django.db.models import Q
 from django.template.loader import render_to_string
 from django.utils.translation import activate
-
+from note.models import Note
-from note.models import NoteUser, Note
 
 
 class Command(BaseCommand):
@@ -17,12 +16,22 @@ class Command(BaseCommand):
         parser.add_argument("--spam", "-s", action='store_true', help="Spam negative users")
         parser.add_argument("--report", "-r", action='store_true', help="Report the list of negative users to admins")
         parser.add_argument("--negative-amount", "-n", action='store', type=int, default=1000,
-                            help="Maximum amount to be considered as very negative")
+                            help="Maximum amount to be considered as very negative (inclusive)")
+        parser.add_argument("--add-years", "-y", action='store', type=int, default=0,
+                            help="Add also people that have a negative balance since N years "
+                                 "(default to only active members)")
 
     def handle(self, *args, **options):
         activate('fr')
+
+        if options['negative_amount'] == 0:
+            # Don't log empty notes
+            options['negative_amount'] = 1
+
         notes = Note.objects.filter(
-            Q(noteuser__user__memberships__date_end__gte=date.today()) | Q(noteclub__isnull=False),
+            Q(noteuser__user__memberships__date_end__gte=
+              date.today() - timedelta(days=int(365.25 * options['add_years'])))
+            | (Q(noteclub__isnull=False) & ~Q(noteclub__club__name__icontains='- BDE')),
             balance__lte=-options["negative_amount"],
             is_active=True, 
         ).order_by('balance').distinct().all()

management/commands/send_reports.py

@@ -1,14 +1,11 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from datetime import timedelta
-
 from django.core.management import BaseCommand
 from django.db.models import Q
 from django.template.loader import render_to_string
 from django.utils import timezone
 from django.utils.translation import activate
-
 from note.models import NoteUser, Transaction
 from note.tables import HistoryTable
 

management/commands/syncsql.py

@@ -1,11 +1,9 @@
-# Copyright (C) 2018-2020 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import connection
-
-
 from polymorphic.models import PolymorphicModel
 
 NO_SEQ = [
@@ -14,6 +12,7 @@ NO_SEQ = [
     "WEIRole",  # dirty fix
 ]
 
+
 class Command(BaseCommand):
     """
     Command to synchronise primary sequence of postgres after bulk insert of django.

shell/.env_borg_example

@@ -0,0 +1,3 @@
+BORG_PASSPHRASE='CHANGE_ME'
+BORG_REPO='USER@SERVER:PATH'
+BACKUP_FILE='PATH'

shell/backup_db

@@ -1,9 +1,14 @@
 #!/bin/bash
+export $(cat .env_borg | xargs)
+
 # Create temporary backups directory
-[[ -d /tmp/note-backups ]] || mkdir /tmp/note-backups
+mkdir -p /tmp/note-backups
-date=$(date +%Y-%m-%d)
+
-# Backup database and save it as tar archive
+# Backup database
-su postgres -c "pg_dump -F t note_db" | tee "/tmp/note-backups/$date.tar" > /dev/null
+sudo -u postgres pg_dump -F t note_db > $BACKUP_FILE
-# Compress backup as gzip
+
-gzip "/tmp/note-backups/$date.tar"
+# Keep the last 30 backups
-scp "/tmp/note-backups/$date.tar.gz" "club-bde@zamok.crans.org:backup/$date.tar.gz"
+borg prune --keep-last 30
+
+# Save backup
+borg create --compression lz4 ::backup-{now} $BACKUP_FILE