send summary script

Remove useless tee, use "mkdir -p" and use .sql.gz.

apps.py

@@ -2,7 +2,6 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.apps import AppConfig
-from django.core.signals import got_request_exception
 
 
 class ScriptsConfig(AppConfig):

management/commands/_import_utils.py

@@ -5,10 +5,9 @@ import json
 import time
 from collections import defaultdict
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import transaction
-
 from polymorphic.models import PolymorphicModel
 
 
@@ -16,6 +15,7 @@ def timed(method):
     """"
     A simple decorator to measure time elapsed in class function (hence the args[0])
     """
+
     def _timed(*args, **kw):
         ts = time.time()
         result = method(*args, **kw)

management/commands/anonymize_data.py

@@ -1,4 +1,4 @@
-# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# Copyright (C) 2018-2023 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 from django.core.management.base import BaseCommand
@@ -7,27 +7,87 @@ from django.db import connection
 
 class Command(BaseCommand):
     """
-    Command to protect sensitive data during the beta phase, to prevent a right escalation.
-    Phone number, email address, postal address, first and last name are removed.
+    Command to protect sensitive data during the beta phase or after WEI.
+    Phone number, email address, postal address, first and last name,
+    IP addresses, health issues, gender and birth date are removed.
     """
     def add_arguments(self, parser):
         parser.add_argument('--force', '-f', action='store_true', help="Actually anonymize data.")
+        parser.add_argument('--type', '-t', choices=["all", "wei", "user"], default="",
+                            help='Select the type of data to anonyze (default None)')
 
-    def handle(self, *args, **kwargs):
-        if not kwargs['force']:
-            self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data with "
-                              "sample data. Don't use this in production! If you know what you are doing, "
+    def handle(self, *args, **options):
+        if not options['force']:
+            if options['type'] == "all":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset ALL personal data with "
+                              "sample data. Don't use  in production! If you know what you are doing, please "
+                              "add --force option.")
+            elif options['type'] == "wei":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset WEI personal data with "
+                              "sample data. Use it in production only after WEI. If you know what you are doing,"
                               "please add --force option.")
+            elif options['type'] == "user":
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data "
+                              "visible by user (not admin or trez BDE) with sample data. Don't use  in "
+                              "production! If you know what you are doing, please add --force option.")
+            else:
+                self.stderr.write("CAUTION: This is a dangerous script. This will reset all personal data with "
+                              "sample data. Don't use  in production ('wei' can be use in production after "
+                              "the WEI)! If you know what you are doing, please choose a type.")
             exit(1)
 
         cur = connection.cursor()
-        cur.execute("UPDATE member_profile SET "
+        if options['type'] in ("all","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize profile, user club and guest data")
+            cur.execute("UPDATE member_profile SET "
                     "phone_number = '0123456789', "
                     "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
-        cur.execute("UPDATE auth_user SET "
+            cur.execute("UPDATE auth_user SET "
                     "first_name = 'Anne', "
                     "last_name = 'Onyme', "
                     "email = 'anonymous@example.com';")
-        cur.execute("UPDATE member_club SET "
+            cur.execute("UPDATE member_club SET "
                     "email = 'anonymous@example.com';")
+            cur.execute("UPDATE activity_guest SET "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+
+        if options['type'] in ("all","wei","user"):
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize WEI data")
+            cur.execute("UPDATE wei_weiregistration SET "
+                    "birth_date = '1998-01-08', "
+                    "emergency_contact_name = 'Anne Onyme', "
+                    "emergency_contact_phone = '0123456789', "
+                    "gender = 'nonbinary', "
+                    "health_issues = 'Tout va bien';")
+
+        if options['type'] == "all":
+            if options['verbosity'] != 0:
+                self.stdout.write("Anonymize invoice, special transaction, log, mailer and oauth data")
+            cur.execute("UPDATE treasury_invoice SET "
+                    "name = 'Anne Onyme', "
+                    "object = 'Rends nous riches', "
+                    "description = 'Donne nous plein de sous', "
+                    "address = '4 avenue des Sciences, 91190 GIF-SUR-YVETTE';")
+            cur.execute("UPDATE treasury_product SET "
+                    "designation = 'un truc inutile';")
+            cur.execute("UPDATE note_specialtransaction SET "
+                    "bank = 'le matelas', "
+                    "first_name = 'Anne', "
+                    "last_name = 'Onyme';")
+            cur.execute("UPDATE logs_changelog SET "
+                    "ip = '127.0.0.1', "
+                    "data = 'new data', "
+                    "previous = 'old data';")
+            cur.execute("UPDATE mailer_messagelog SET "
+                    "log_message = 'log message', "
+                    "message_data = 'message data';")
+            cur.execute("UPDATE mailer_dontsendentry SET "
+                    "to_address = 'anonymous@example.com';")
+            cur.execute("UPDATE oauth2_provider_application SET "
+                    "name = 'external app', "
+                    "redirect_uris = 'http://external.app', "
+                    "client_secret = 'abcdefghijklmnopqrstuvwxyz';")
         cur.close()

management/commands/compilejsmessages.py

@@ -13,6 +13,7 @@ class Command(BaseCommand):
     """
     Generate Javascript translation files
     """
+
     def handle(self, *args, **kwargs):
         for code, _ in settings.LANGUAGES:
             if code == settings.LANGUAGE_CODE:

management/commands/extract_ml_registrations.py

@@ -5,14 +5,12 @@ from datetime import date
 
 from django.contrib.auth.models import User
 from django.core.management import BaseCommand
-from django.db.models import Q
-from member.models import Membership, Club
-from wei.models import WEIClub
+from member.models import Club, Membership
 
 
 class Command(BaseCommand):
     help = "Get mailing list registrations from the last wei. " \
-           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -t {fr, en}. " \
+           "Usage: manage.py extract_ml_registrations -t {events,art,sport} -t {fr, en} -y {0, 1, ...}. " \
            "You can write this into a file with a pipe, then paste the document into your mail manager."
 
     def add_arguments(self, parser):
@@ -21,6 +19,8 @@ class Command(BaseCommand):
         parser.add_argument('--lang', '-l', type=str, choices=['fr', 'en'], default='fr',
                             help='Select the registred users of the ML of the given language. Useful only for the '
                                  'events mailing list.')
+        parser.add_argument('--years', '-y', type=int, default=0, 
+                            help='Select the cumulative registred users of a membership from years ago. 0 means the current users')
 
     def handle(self, *args, **options):
         # TODO: Improve the mailing list extraction system, and link it automatically with Mailman.
@@ -30,10 +30,12 @@ class Command(BaseCommand):
             return
 
         if options["type"] == "members":
+            today_date = date.today()
+            selected_date = date(today_date.year - options["years"], today_date.month, today_date.day)
             for membership in Membership.objects.filter(
                 club__name="BDE",
-                date_start__lte=date.today(),
-                date_end__gte=date.today(),
+                date_start__lte=today_date,
+                date_end__gte=selected_date,
             ).all():
                 self.stdout.write(membership.user.email)
             return

management/commands/force_delete_user.py

@@ -11,7 +11,6 @@ from django.core.management.base import BaseCommand
 from django.db import transaction
 from django.db.models import Q
 from django.test import override_settings
-
 from note.models import Alias, Transaction
 
 

management/commands/import_account.py

@@ -1,23 +1,19 @@
 # Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import psycopg2 as pg
-import psycopg2.extras as pge
 import datetime
 import json
 
-from django.template.loader import render_to_string
-from django.utils.timezone import make_aware, now
+import psycopg2 as pg
+import psycopg2.extras as pge
 from django.contrib.auth.models import User
 from django.contrib.contenttypes.models import ContentType
-
 from django.db import transaction
-
-from note.models import Note, NoteUser, NoteClub
-from note.models import Alias
+from django.utils.timezone import make_aware
 from member.models import Club, Profile
+from note.models import Alias, Note, NoteClub, NoteUser
 
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 M_DURATION = 396
 M_START = datetime.date(2019, 8, 1)

management/commands/import_activities.py

@@ -3,32 +3,30 @@
 
 import psycopg2 as pg
 import psycopg2.extras as pge
-import datetime
-import copy
-
-from django.utils.timezone import make_aware
+from activity.models import Activity, ActivityType, Entry, Guest
 from django.db import transaction
-
-from activity.models import ActivityType, Activity, Guest, Entry
+from django.utils.timezone import make_aware
 from member.models import Club
 from note.models import Note, NoteUser
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_ACTIVITY = dict()
 
 CLUB_RELOU = [
-    0,    # BDE
-    4771, # Kataclist
-    5162, # Assurance BDE ?!
-    5164, # S & L
+    0,  # BDE
+    4771,  # Kataclist
+    5162,  # Assurance BDE ?!
+    5164,  # S & L
     625,  # Aspique
-    5154, # Frekens
-    3944, # DiskJok[ENS]
-    5153, # Monopo[list]
-    2351, # JdRM
-    2365, # Pot Vieux
+    5154,  # Frekens
+    3944,  # DiskJok[ENS]
+    5153,  # Monopo[list]
+    2351,  # JdRM
+    2365,  # Pot Vieux
 ]
 
+
 class Command(ImportCommand):
     """
     Import command for Activities Base Data (Comptes, and Aliases)
@@ -50,7 +48,7 @@ class Command(ImportCommand):
                 row["responsable"] = 3508
             note = self.MAP_IDBDE[row["responsable"]]
             if note == 6244:
-                 # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
+                # Licorne magique ne doit pas utiliser son compte club pour proposer des activités
                 note = Note.objects.get(pk=self.MAP_IDBDE[6524])
                 note = note.id
             organizer = Club.objects.filter(name=row["signature"])

management/commands/import_nk15.py

@@ -3,15 +3,15 @@
 
 import subprocess
 
-from django.core.management.base import BaseCommand
 from django.core.management import call_command
 
 from ._import_utils import ImportCommand
 
+
 class Command(ImportCommand):
     """
     Command for importing the database of NK15.
-    Need to be run by a user with a registered role in postgres for the database nk15. 
+    Need to be run by a user with a registered role in postgres for the database nk15.
     """
 
     def handle(self, *args, **kwargs):

management/commands/import_transaction.py

@@ -1,31 +1,25 @@
 # Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
+import copy
+import datetime
 import re
+
+import pytz
 import psycopg2 as pg
 import psycopg2.extras as pge
-import pytz
-import datetime
-import copy
-
+from activity.models import Entry, GuestTransaction
 from django.contrib.auth.models import User
-from django.utils.timezone import make_aware
-from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-
-from note.models import (TemplateCategory,
-                         TransactionTemplate,
-                         Transaction,
-                         RecurrentTransaction,
-                         SpecialTransaction,
-                         MembershipTransaction,
-                         )
-from note.models import Note, NoteClub
-from activity.models import Guest, GuestTransaction, Entry
-
+from django.db import transaction
+from django.utils.timezone import make_aware
 from member.models import Membership
-from treasury.models import Remittance, SpecialTransactionProxy, SogeCredit
-from ._import_utils import ImportCommand, BulkCreateManager, timed
+from note.models import (MembershipTransaction, Note, NoteClub,
+                         RecurrentTransaction, SpecialTransaction,
+                         TemplateCategory, Transaction, TransactionTemplate)
+from treasury.models import Remittance, SogeCredit, SpecialTransactionProxy
+
+from ._import_utils import BulkCreateManager, ImportCommand, timed
 
 MAP_TRANSACTION = dict()
 MAP_REMITTANCE = dict()

management/commands/refresh_activities.py

@@ -2,14 +2,14 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import os
-from bs4 import BeautifulSoup
-from django.core.management import BaseCommand
-from django.urls import reverse
-from django.utils import timezone
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 
+from bs4 import BeautifulSoup
 from activity.models import Activity
+from django.core.management import BaseCommand
+from django.urls import reverse
+from django.utils import timezone
 
 
 class Command(BaseCommand):

management/commands/refresh_highlighted_buttons.py

@@ -6,7 +6,6 @@ from datetime import timedelta
 from django.core.management.base import BaseCommand
 from django.db.models import Count
 from django.utils import timezone
-
 from note.models import RecurrentTransaction, TransactionTemplate
 
 

management/commands/send_mail_to_negative_balances.py

@@ -8,8 +8,7 @@ from django.core.management import BaseCommand
 from django.db.models import Q
 from django.template.loader import render_to_string
 from django.utils.translation import activate
-
-from note.models import NoteUser, Note
+from note.models import Note
 
 
 class Command(BaseCommand):

management/commands/send_reports.py

@@ -1,14 +1,11 @@
 # Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from datetime import timedelta
-
 from django.core.management import BaseCommand
 from django.db.models import Q
 from django.template.loader import render_to_string
 from django.utils import timezone
 from django.utils.translation import activate
-
 from note.models import NoteUser, Transaction
 from note.tables import HistoryTable
 

management/commands/send_summary_notes_report.py

@@ -0,0 +1,144 @@
+# Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from datetime import date
+
+from django.core.mail import send_mail
+from django.core.management import BaseCommand
+from django.db.models import Q
+from django.template.loader import render_to_string
+from django.utils.translation import activate
+from note.models import NoteUser, NoteClub
+from treasury.models import NoteSummary
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--negative-amount", "-n", action='store', type=int, default=1000,
+                            help="Maximum amount to be considered as very negative (inclusive)")
+
+    def handle(self, *args, **options):
+        activate('fr')
+
+        if options['negative_amount'] == 0:
+            # Don't log empty notes
+            options['negative_amount'] = 1
+    
+        
+        # User notes
+        positive_user_notes = NoteUser.objects.filter( Q(balance__gt=0) ).distinct()
+        positive_user_notes_bde = positive_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        zero_user_notes = NoteUser.objects.filter( Q(balance=0) ).distinct()
+        zero_user_notes_bde = zero_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        negative_user_notes = NoteUser.objects.filter( Q(balance__lt=0) ).distinct()
+        negative_user_notes_bde = negative_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+        vnegative_user_notes = NoteUser.objects.filter( Q(balance__lte=-options["negative_amount"]) ).distinct()
+        vnegative_user_notes_bde = vnegative_user_notes.filter( Q(user__memberships__club__name = 'BDE') & Q(user__memberships__date_end__gte = date.today()) ).distinct()
+
+
+        total_positive_user = positive_user_notes.count()
+        balance_positive_user = sum(note.balance for note in positive_user_notes.all())
+
+        total_positive_user_bde = positive_user_notes_bde.count()
+        balance_positive_user_bde = sum(note.balance for note in positive_user_notes_bde.all())
+
+        total_zero_user = zero_user_notes.count()
+
+        total_zero_user_bde = zero_user_notes_bde.count()
+
+        total_negative_user = negative_user_notes.count()
+        balance_negative_user = sum(note.balance for note in negative_user_notes.all())
+
+        total_negative_user_bde = negative_user_notes_bde.count()
+        balance_negative_user_bde = sum(note.balance for note in negative_user_notes_bde.all())
+
+        total_vnegative_user = vnegative_user_notes.count()
+        balance_vnegative_user = sum(note.balance for note in vnegative_user_notes.all())
+
+        total_vnegative_user_bde = vnegative_user_notes_bde.count()
+        balance_vnegative_user_bde = sum(note.balance for note in vnegative_user_notes_bde.all())
+
+        
+
+        #Club notes
+        positive_club_notes = NoteClub.objects.filter( Q(balance__gt=0) ).distinct()
+        positive_club_notes_nbde = positive_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+        zero_club_notes = NoteClub.objects.filter( Q(balance=0) ).distinct()
+        zero_club_notes_nbde = zero_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+        negative_club_notes = NoteClub.objects.filter( Q(balance__lt=0) ).distinct()
+        negative_club_notes_nbde = negative_club_notes.filter( ~Q(club__name = 'BDE') & ~Q(club__name = 'Kfet') & ~Q(club__name__iendswith = '- BDE')).distinct()
+
+
+        total_positive_club = positive_club_notes.count()
+        balance_positive_club = sum(note.balance for note in positive_club_notes.all())
+
+        total_positive_club_nbde = positive_club_notes_nbde.count()
+        balance_positive_club_nbde = sum(note.balance for note in positive_club_notes_nbde.all())
+
+        total_zero_club = zero_club_notes.count()
+
+        total_zero_club_nbde = zero_club_notes_nbde.count()
+
+        total_negative_club = negative_club_notes.count()
+        balance_negative_club = sum(note.balance for note in negative_club_notes.all())
+
+        total_negative_club_nbde = negative_club_notes_nbde.count()
+        balance_negative_club_nbde = sum(note.balance for note in negative_club_notes_nbde.all())
+        
+
+        last_summary = NoteSummary.objects.order_by('-date').first()
+
+        summary = NoteSummary.objects.create(
+            total_positive_user=total_positive_user,
+            balance_positive_user=balance_positive_user,
+
+            total_positive_user_bde=total_positive_user_bde,
+            balance_positive_user_bde=balance_positive_user_bde,
+
+            total_zero_user=total_zero_user,
+
+            total_zero_user_bde=total_zero_user_bde,
+
+            total_negative_user=total_negative_user,
+            balance_negative_user=balance_negative_user,
+
+            total_negative_user_bde=total_negative_user_bde,
+            balance_negative_user_bde=balance_negative_user_bde,
+
+            total_vnegative_user=total_vnegative_user,
+            balance_vnegative_user=balance_vnegative_user,
+
+            total_vnegative_user_bde=total_vnegative_user_bde,
+            balance_vnegative_user_bde=balance_vnegative_user_bde,
+
+
+            total_positive_club=total_positive_club,
+            balance_positive_club=balance_positive_club,
+
+            total_positive_club_nbde=total_positive_club_nbde,
+            balance_positive_club_nbde=balance_positive_club_nbde,
+
+            total_zero_club=total_zero_club,
+
+            total_zero_club_nbde=total_zero_club_nbde,
+
+            total_negative_club=total_negative_club,
+            balance_negative_club=balance_negative_club,
+
+            total_negative_club_nbde=total_negative_club_nbde,
+            balance_negative_club_nbde=balance_negative_club_nbde,
+        )
+
+        balance_difference_user = (balance_positive_user - balance_negative_user) - (last_summary.balance_positive_user - last_summary.balance_negative_user)
+        balance_difference_club = (balance_positive_club - balance_negative_club) - (last_summary.balance_positive_club - last_summary.balance_negative_club)
+
+        plain_text = render_to_string("note/mails/summary_notes_report.txt", context=dict(summary=summary, balance_difference_user=balance_difference_user, balance_difference_club=balance_difference_club))
+        html = render_to_string("note/mails/summary_notes_report.html", context=dict(summary=summary, balance_difference_user=balance_difference_user, balance_difference_club=balance_difference_club))
+        send_mail("[Note Kfet] Récapitulatif de trésorerie", plain_text, "Note Kfet 2020 <notekfet2020@crans.org>",
+                  recipient_list=["respo-info.bde@lists.crans.org", "tresorerie.bde@lists.crans.org"],
+                  html_message=html)

management/commands/syncsql.py

@@ -1,11 +1,9 @@
 # Copyright (C) 2018-2021 by BDE ENS Paris-Saclay
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-from django.core.management.base import BaseCommand
 from django.apps import apps
+from django.core.management.base import BaseCommand
 from django.db import connection
-
-
 from polymorphic.models import PolymorphicModel
 
 NO_SEQ = [
@@ -14,6 +12,7 @@ NO_SEQ = [
     "WEIRole",  # dirty fix
 ]
 
+
 class Command(BaseCommand):
     """
     Command to synchronise primary sequence of postgres after bulk insert of django.

shell/backup_db

@@ -1,9 +1,9 @@
 #!/bin/bash
 # Create temporary backups directory
-[[ -d /tmp/note-backups ]] || mkdir /tmp/note-backups
+mkdir -p /tmp/note-backups
 date=$(date +%Y-%m-%d)
 # Backup database and save it as tar archive
-su postgres -c "pg_dump -F t note_db" | tee "/tmp/note-backups/$date.tar" > /dev/null
+sudo -u postgres pg_dump -F t note_db > "/tmp/note-backups/$date.sql"
 # Compress backup as gzip
-gzip "/tmp/note-backups/$date.tar"
-scp "/tmp/note-backups/$date.tar.gz" "club-bde@zamok.crans.org:backup/$date.tar.gz"
+gzip "/tmp/note-backups/$date.sql"
+scp "/tmp/note-backups/$date.sql.gz" "club-bde@zamok.crans.org:backup/$date.sql.gz"