From 6f780c3f276941d15f95183dc31c8e50a4dd509f Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 14:07:39 +0100 Subject: [PATCH 01/79] Remove sporz app --- med/settings.py | 1 - sporz/__init__.py | 5 - sporz/admin.py | 76 --------------- sporz/apps.py | 11 --- sporz/locale/fr/LC_MESSAGES/django.po | 134 -------------------------- sporz/migrations/0001_initial.py | 51 ---------- sporz/migrations/__init__.py | 0 sporz/models.py | 117 ---------------------- 8 files changed, 395 deletions(-) delete mode 100644 sporz/__init__.py delete mode 100644 sporz/admin.py delete mode 100644 sporz/apps.py delete mode 100644 sporz/locale/fr/LC_MESSAGES/django.po delete mode 100644 sporz/migrations/0001_initial.py delete mode 100644 sporz/migrations/__init__.py delete mode 100644 sporz/models.py diff --git a/med/settings.py b/med/settings.py index 850958c..865a6b2 100644 --- a/med/settings.py +++ b/med/settings.py @@ -51,7 +51,6 @@ INSTALLED_APPS = [ 'med', 'media', 'logs', - 'sporz', ] MIDDLEWARE = [ diff --git a/sporz/__init__.py b/sporz/__init__.py deleted file mode 100644 index adac51a..0000000 --- a/sporz/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright (C) 2017-2019 by BDE ENS Paris-Saclay -# SPDX-License-Identifier: GPL-3.0-or-later - -default_app_config = 'sporz.apps.SporzConfig' diff --git a/sporz/admin.py b/sporz/admin.py deleted file mode 100644 index 06e2fc2..0000000 --- a/sporz/admin.py +++ /dev/null @@ -1,76 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright (C) 2017-2019 by BDE ENS Paris-Saclay -# SPDX-License-Identifier: GPL-3.0-or-later - -from django.contrib import admin -from django.contrib.auth import get_user_model -from django.db.models import Q - -from med.admin import admin_site -from .models import GameSave, Player - - -class PlayerInline(admin.TabularInline): - model = Player - - # Do not always show extra players - extra = 0 - min_num = 5 - - -class GameSaveAdmin(admin.ModelAdmin): - inlines = [PlayerInline, ] - list_display = ('__str__', 'game_master', 'game_has_ended') - date_hierarchy = 'created_at' - autocomplete_fields = ('game_master',) - - def has_change_permission(self, request, obj=None): - """ - If user is game master then authorize edit - """ - if obj and obj.game_master == request.user: - return True - return super().has_change_permission(request, obj) - - def has_delete_permission(self, request, obj=None): - """ - If user is game master then authorize deletion - """ - if obj and obj.game_master == request.user: - return True - return super().has_delete_permission(request, obj) - - def add_view(self, request, form_url='', extra_context=None): - """ - Autoselect game master when creating a new game - """ - # Make GET data mutable - data = request.GET.copy() - data['game_master'] = request.user - request.GET = data - return super().add_view(request, form_url, extra_context) - - def formfield_for_foreignkey(self, db_field, request, **kwargs): - """ - Authorize game master change only if user can see all users - """ - if db_field.name == 'game_master': - if not request.user.has_perm('users.view_user'): - kwargs['queryset'] = get_user_model().objects.filter( - username=request.user.username) - return super().formfield_for_foreignkey(db_field, request, **kwargs) - - def get_queryset(self, request): - """ - List all game save only if user has view permission - else, list only own games and ended games - """ - queryset = super().get_queryset(request) - if request.user.has_perm('sporz.view_gamesave'): - return queryset - return queryset.filter( - Q(game_master=request.user) | Q(game_has_ended=True) - ) - - -admin_site.register(GameSave, GameSaveAdmin) diff --git a/sporz/apps.py b/sporz/apps.py deleted file mode 100644 index 33fcffb..0000000 --- a/sporz/apps.py +++ /dev/null @@ -1,11 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright (C) 2017-2019 by BDE ENS Paris-Saclay -# SPDX-License-Identifier: GPL-3.0-or-later - -from django.apps import AppConfig -from django.utils.translation import ugettext_lazy as _ - - -class SporzConfig(AppConfig): - name = 'sporz' - verbose_name = _('Sporz game assitant') diff --git a/sporz/locale/fr/LC_MESSAGES/django.po b/sporz/locale/fr/LC_MESSAGES/django.po deleted file mode 100644 index 793f947..0000000 --- a/sporz/locale/fr/LC_MESSAGES/django.po +++ /dev/null @@ -1,134 +0,0 @@ -#, fuzzy -msgid "" -msgstr "" -"Project-Id-Version: PACKAGE VERSION\n" -"Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2019-08-15 11:29+0200\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" -"Language-Team: LANGUAGE \n" -"Language: \n" -"MIME-Version: 1.0\n" -"Content-Type: text/plain; charset=UTF-8\n" -"Content-Transfer-Encoding: 8bit\n" -"Plural-Forms: nplurals=2; plural=(n > 1);\n" - -#: apps.py:11 -msgid "Sporz game assitant" -msgstr "Assitant au jeu Sporz" - -#: models.py:13 -msgid "created at" -msgstr "créé le" - -#: models.py:20 -msgid "game master" -msgstr "maître du jeu" - -#: models.py:21 -msgid "Game master can edit and delete this game save." -msgstr "Le maître du jeu peut éditer et supprimer cette sauvegarde." - -#: models.py:24 -msgid "current round" -msgstr "tour actuel" - -#: models.py:28 -msgid "game has ended" -msgstr "la partie est finie" - -#: models.py:29 -msgid "If true, then everyone will be able to see the game." -msgstr "Quand cette case est cochée, tout le monde pourra voir le récapitulatif." - -#: models.py:36 models.py:115 -msgid "players" -msgstr "joueurs" - -#: models.py:40 -msgid "game save" -msgstr "sauvegarde de jeu" - -#: models.py:41 -msgid "game saves" -msgstr "sauvegardes de jeu" - -#: models.py:58 -msgid "Base astronaut" -msgstr "Astronaute de base" - -#: models.py:59 -msgid "Base mutant" -msgstr "Mutant de base" - -#: models.py:60 -msgid "Healer" -msgstr "Médecin" - -#: models.py:61 -msgid "Psychologist" -msgstr "Psychologue" - -#: models.py:62 -msgid "Geno-technician" -msgstr "Geno-technicien" - -#: models.py:63 -msgid "Computer scientist" -msgstr "Informaticien" - -#: models.py:64 -msgid "Hacker" -msgstr "Hackeur" - -#: models.py:65 -msgid "Spy" -msgstr "Espion" - -#: models.py:66 -msgid "Detective" -msgstr "Enquêteur" - -#: models.py:67 -msgid "Traitor" -msgstr "Traître" - -#: models.py:75 -msgid "Neutral" -msgstr "Neutre" - -#: models.py:76 -msgid "Host" -msgstr "Hôte" - -#: models.py:77 -msgid "Immunized" -msgstr "Immunisé" - -#: models.py:83 -msgid "game" -msgstr "jeu" - -#: models.py:87 -msgid "name" -msgstr "nom" - -#: models.py:94 -msgid "user" -msgstr "utilisateur" - -#: models.py:95 -msgid "Optionnal mapping to an user." -msgstr "Lien optionnel à un utilisateur." - -#: models.py:103 -msgid "genotype" -msgstr "génotype" - -#: models.py:107 -msgid "infected" -msgstr "infecté" - -#: models.py:114 -msgid "player" -msgstr "joueur" diff --git a/sporz/migrations/0001_initial.py b/sporz/migrations/0001_initial.py deleted file mode 100644 index 69e18d0..0000000 --- a/sporz/migrations/0001_initial.py +++ /dev/null @@ -1,51 +0,0 @@ -# Generated by Django 2.2.4 on 2019-08-15 09:31 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import django.utils.timezone - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name='GameSave', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('created_at', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='created at')), - ('current_round', models.PositiveSmallIntegerField(default=1, verbose_name='current round')), - ('game_has_ended', models.BooleanField(help_text='If true, then everyone will be able to see the game.', verbose_name='game has ended')), - ('game_master', models.ForeignKey(help_text='Game master can edit and delete this game save.', on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='game master')), - ], - options={ - 'verbose_name': 'game save', - 'verbose_name_plural': 'game saves', - 'ordering': ['-created_at'], - }, - ), - migrations.CreateModel( - name='Player', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.CharField(max_length=150, verbose_name='name')), - ('role', models.CharField(choices=[('BA', 'Base astronaut'), ('BM', 'Base mutant'), ('HE', 'Healer'), ('PS', 'Psychologist'), ('GE', 'Geno-technician'), ('CO', 'Computer scientist'), ('HA', 'Hacker'), ('SP', 'Spy'), ('DE', 'Detective'), ('TR', 'Traitor')], default='BA', max_length=2)), - ('genotype', models.NullBooleanField(choices=[(None, 'Neutral'), (False, 'Host'), (True, 'Immunized')], verbose_name='genotype')), - ('infected', models.BooleanField(verbose_name='infected')), - ('game', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sporz.GameSave', verbose_name='game')), - ('user', models.ForeignKey(blank=True, help_text='Optionnal mapping to an user.', null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL, verbose_name='user')), - ], - options={ - 'verbose_name': 'player', - 'verbose_name_plural': 'players', - 'ordering': ['user__username'], - 'unique_together': {('game', 'name')}, - }, - ), - ] diff --git a/sporz/migrations/__init__.py b/sporz/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/sporz/models.py b/sporz/models.py deleted file mode 100644 index 926b88b..0000000 --- a/sporz/models.py +++ /dev/null @@ -1,117 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright (C) 2017-2019 by BDE ENS Paris-Saclay -# SPDX-License-Identifier: GPL-3.0-or-later - -from django.conf import settings -from django.db import models -from django.utils import timezone -from django.utils.translation import gettext_lazy as _ - - -class GameSave(models.Model): - created_at = models.DateTimeField( - verbose_name=_('created at'), - default=timezone.now, - editable=False, - ) - game_master = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.CASCADE, - verbose_name=_('game master'), - help_text=_('Game master can edit and delete this game save.'), - ) - current_round = models.PositiveSmallIntegerField( - verbose_name=_('current round'), - default=1, - ) - game_has_ended = models.BooleanField( - verbose_name=_('game has ended'), - help_text=_('If true, then everyone will be able to see the game.'), - ) - - def __str__(self): - return "{} ({} {})".format( - self.created_at.strftime("%b %d %Y %H:%M:%S"), - len(self.player_set.all()), - _("players"), - ) - - class Meta: - verbose_name = _("game save") - verbose_name_plural = _("game saves") - ordering = ['-created_at'] - - -class Player(models.Model): - # Player roles - BASE_ASTRONAUT = 'BA' - BASE_MUTANT = 'BM' - HEALER = 'HE' - PSYCHOLOGIST = 'PS' - GENO_TECHNICIAN = 'GE' - COMPUTER_SCIENTIST = 'CO' - HACKER = 'HA' - SPY = 'SP' - DETECTIVE = 'DE' - TRAITOR = 'TR' - ROLES = [ - (BASE_ASTRONAUT, _('Base astronaut')), - (BASE_MUTANT, _("Base mutant")), - (HEALER, _("Healer")), - (PSYCHOLOGIST, _("Psychologist")), - (GENO_TECHNICIAN, _("Geno-technician")), - (COMPUTER_SCIENTIST, _("Computer scientist")), - (HACKER, _("Hacker")), - (SPY, _("Spy")), - (DETECTIVE, _("Detective")), - (TRAITOR, _("Traitor")), - ] - - # Genotypes - NEUTRAL = None - HOST = False - IMMUNIZED = True - GENOTYPES = [ - (NEUTRAL, _("Neutral")), - (HOST, _("Host")), - (IMMUNIZED, _("Immunized")) - ] - - game = models.ForeignKey( - GameSave, - on_delete=models.CASCADE, - verbose_name=_('game'), - ) - name = models.CharField( - max_length=150, - verbose_name=_('name'), - ) - user = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.CASCADE, - blank=True, - null=True, - verbose_name=_('user'), - help_text=_('Optionnal mapping to an user.'), - ) - role = models.CharField( - max_length=2, - choices=ROLES, - default=BASE_ASTRONAUT, - ) - genotype = models.NullBooleanField( - verbose_name=_('genotype'), - choices=GENOTYPES, - ) - infected = models.BooleanField( - verbose_name=_('infected'), - ) - - def __str__(self): - return str(self.name) - - class Meta: - verbose_name = _("player") - verbose_name_plural = _("players") - ordering = ['user__username'] - unique_together = ['game', 'name'] From 4d8d54e7de2bb045c81829f549401ef60e6070f3 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 14:18:39 +0100 Subject: [PATCH 02/79] Add uwsgi configuration --- README.md | 38 ++++++++++++++++++++++++++++++++------ entrypoint.sh | 21 +++++++++++++++++++-- requirements.txt | 3 ++- 3 files changed, 53 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index c398e50..0b3b323 100644 --- a/README.md +++ b/README.md @@ -11,22 +11,48 @@ Elle permet de gérer les medias, bd, jeux, emprunts, ainsi que les adhérents d Ce projet est sous la licence GNU public license v3.0. -## Développement +## Installation -Après avoir installé un environnement Django, +### Développement + +On peut soit développer avec Docker, soit utiliser un VirtualEnv. + +Dans le cas du VirtualEnv, ```bash +python3 -m venv venv +. venv/bin/activate +pip install -r requirements.txt ./manage.py migrate ./manage.py collectstatic ./manage.py runserver ``` -## Configuration d'une base MySQL +### Production -Sur le serveur mysql ou postgresl, il est nécessaire de créer une base de donnée med, +Vous pouvez soit utiliser Docker, soit configurer manuellement le serveur. + +#### Mise en place du projet sur Zamok + +Pour mettre en place le projet sans droits root, +on va créer un socket uwsgi dans le répertoire personnel de l'utilisateur `club-med` +puis on va dire à Apache2 d'utiliser ce socket avec un `.htaccess`. + +Pour cela on va imiter ce que fait l'image Docker, + +```bash +python3 -m venv venv +. venv/bin/activate +pip install -r requirements.txt +./entrypoint.sh +``` + +#### Configuration d'une base de données + +Sur le serveur MySQL ou PostgreSQL, il est nécessaire de créer une base de donnée med, ainsi qu'un user med et un mot de passe associé. -Voici les étapes à éxecuter pour mysql : +Voici les étapes à executer pour MySQL : ```SQL CREATE DATABASE med; @@ -35,7 +61,7 @@ GRANT ALL PRIVILEGES ON med.* TO 'med'@'localhost'; FLUSH PRIVILEGES; ``` -Et pour postgresql : +Et pour PostgreSQL : ```SQL CREATE DATABASE med; diff --git a/entrypoint.sh b/entrypoint.sh index 3af5810..5538c65 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,8 +1,25 @@ #!/bin/bash +# This will launch the Django project as a fastcgi socket +# then Apache or NGINX will be able to use that socket + python manage.py compilemessages python manage.py makemigrations + +# Wait for database sleep 2 + python manage.py migrate -# TODO: use uwsgi in production -python manage.py runserver 0.0.0.0:8000 +uwsgi --chdir="$(pwd)" \ + --module=med.wsgi:application \ + --env DJANGO_SETTINGS_MODULE=med.settings \ + --master \ + --pidfile="$(pwd)/uwsgi.pid" \ # create a pidfile + --socket="$(pwd)/uwsgi.sock" \ + --processes=5 \ + --chmod-socket=600 \ + --harakiri=20 \ # respawn processes taking more than 20 seconds + --max-requests=5000 \ # respawn processes after serving 5000 requests + --vacuum \ # clean up when stopped + --daemonize="$(pwd)/uwsgi.log" \ + --protocol=fastcgi \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 651bd3a..31b84ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,5 @@ python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2 +psycopg2==2.7.7 +uwsgi==2.0.18 \ No newline at end of file From 8a13f87a9e64f6151c0d603737901461e0a22367 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:08:27 +0100 Subject: [PATCH 03/79] Clarify Django commands --- README.md | 3 ++- entrypoint.sh | 1 + start_uwsgi.sh | 19 ------------------- 3 files changed, 3 insertions(+), 20 deletions(-) delete mode 100644 start_uwsgi.sh diff --git a/README.md b/README.md index 0b3b323..3ca63c7 100644 --- a/README.md +++ b/README.md @@ -23,8 +23,9 @@ Dans le cas du VirtualEnv, python3 -m venv venv . venv/bin/activate pip install -r requirements.txt +./manage.py compilemessages +./manage.py makemigrations ./manage.py migrate -./manage.py collectstatic ./manage.py runserver ``` diff --git a/entrypoint.sh b/entrypoint.sh index 5538c65..456da97 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,6 +9,7 @@ python manage.py makemigrations sleep 2 python manage.py migrate +python manage.py collectstatic uwsgi --chdir="$(pwd)" \ --module=med.wsgi:application \ diff --git a/start_uwsgi.sh b/start_uwsgi.sh deleted file mode 100644 index fa88edd..0000000 --- a/start_uwsgi.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -# This will launch the Django project as a UWSGI socket -# then Apache or NGINX will be able to use that socket - -PROJECT_PATH="$(pwd)" - -# Official Django configuration -uwsgi_python3 --chdir=$PROJECT_PATH \ - --module=med.wsgi:application \ - --env DJANGO_SETTINGS_MODULE=med.settings \ - --master --pidfile=$PROJECT_PATH/uwsgi.pid \ - --socket=$PROJECT_PATH/uwsgi.sock \ - --processes=5 \ - --chmod-socket=600 \ - --harakiri=20 \ - --max-requests=5000 \ - --vacuum \ - --daemonize=$PROJECT_PATH/uwsgi.log \ - --protocol=fastcgi From 5c3c8eed8e793f4e224f8d345ee1162737a5a93c Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:08:52 +0100 Subject: [PATCH 04/79] Do not cover dead code --- .coveragerc | 2 -- 1 file changed, 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index d842a7b..a092963 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,9 +3,7 @@ source = logs med media - search static - templates theme users omit = From 32dbf748a12d54513ef95c5e2e65b048a910d9d6 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:09:18 +0100 Subject: [PATCH 05/79] Use sane defaults for development --- med/settings.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/med/settings.py b/med/settings.py index 865a6b2..3f24651 100644 --- a/med/settings.py +++ b/med/settings.py @@ -16,7 +16,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = 'CHANGE_ME_IN_LOCAL_SETTINGS!' # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = False +DEBUG = True ADMINS = ( # ('Admin', 'webmaster@example.com'), @@ -144,7 +144,7 @@ USE_TZ = True # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. # Example: "/var/www/example.com/static/" -STATIC_ROOT = os.path.join(BASE_DIR, 'static_files') +STATIC_ROOT = os.path.join(BASE_DIR, 'static') # URL prefix for static files. # Example: "http://example.com/static/", "http://static.example.com/" @@ -152,6 +152,8 @@ STATIC_URL = '/static/' # Django REST Framework REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination', + 'PAGE_SIZE': 10, 'DEFAULT_PERMISSION_CLASSES': [ 'med.permissions.DjangoViewModelPermissions', ] From 3337c70a21b51a0bc7e95022219c496ed734f26e Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:26:34 +0100 Subject: [PATCH 06/79] Fix entrypoint --- README.md | 13 +++++++++++++ entrypoint.sh | 11 +++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3ca63c7..c53fc42 100644 --- a/README.md +++ b/README.md @@ -42,12 +42,25 @@ puis on va dire à Apache2 d'utiliser ce socket avec un `.htaccess`. Pour cela on va imiter ce que fait l'image Docker, ```bash +git clone https://gitlab.crans.org/mediatek/med.git django-med +chmod go-rwx -R django-med python3 -m venv venv . venv/bin/activate pip install -r requirements.txt ./entrypoint.sh ``` +Pour lancer le serveur au démarrage de Zamok, +on ajoute dans la crontab de l'utilisateur club-med (`crontab -e`) +la ligne suivante : + +```crontab +@reboot /home/club-med/django-med/entrypoint.sh +``` + +Il est néanmoins une mauvaise idée de faire de la production sur SQLite, +on configure donc ensuite Django et une base de données. + #### Configuration d'une base de données Sur le serveur MySQL ou PostgreSQL, il est nécessaire de créer une base de donnée med, diff --git a/entrypoint.sh b/entrypoint.sh index 456da97..58c346f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,16 +11,19 @@ sleep 2 python manage.py migrate python manage.py collectstatic +# harakiri parameter respawns processes taking more than 20 seconds +# max-requests parameter respawns processes after serving 5000 requests +# vacuum parameter cleans up when stopped uwsgi --chdir="$(pwd)" \ --module=med.wsgi:application \ --env DJANGO_SETTINGS_MODULE=med.settings \ --master \ - --pidfile="$(pwd)/uwsgi.pid" \ # create a pidfile + --pidfile="$(pwd)/uwsgi.pid" \ --socket="$(pwd)/uwsgi.sock" \ --processes=5 \ --chmod-socket=600 \ - --harakiri=20 \ # respawn processes taking more than 20 seconds - --max-requests=5000 \ # respawn processes after serving 5000 requests - --vacuum \ # clean up when stopped + --harakiri=20 \ + --max-requests=5000 \ + --vacuum \ --daemonize="$(pwd)/uwsgi.log" \ --protocol=fastcgi \ No newline at end of file From d7d68609d1145cc042cd446567ec4012aca84015 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:29:05 +0100 Subject: [PATCH 07/79] No input when collecting statics --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 58c346f..aa2d908 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,7 +9,7 @@ python manage.py makemigrations sleep 2 python manage.py migrate -python manage.py collectstatic +python manage.py collectstatic --no-input # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests From 3fe0dfbc0244943143f1a3bc3d34d55cbae4b45d Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 15:45:49 +0100 Subject: [PATCH 08/79] How to kill server --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c53fc42..2fde290 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ la ligne suivante : @reboot /home/club-med/django-med/entrypoint.sh ``` +Pour couper le serveur, on tue le maître UWSGI, + +```bash +kill -INT `cat ~/django-med/uwsgi.pid` +``` + Il est néanmoins une mauvaise idée de faire de la production sur SQLite, on configure donc ensuite Django et une base de données. From 754b9632c3e473fc62f7fadeabdb98f121fffd1c Mon Sep 17 00:00:00 2001 From: Yohann D'anello Date: Sun, 9 Feb 2020 16:02:19 +0100 Subject: [PATCH 09/79] README Apache --- .gitignore | 3 ++- README.md | 13 +++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d527d8c..5f4361a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,8 +33,9 @@ coverage # Local data settings_local.py -static_files/* +static/* *.log +*.pid # Virtualenv env/ diff --git a/README.md b/README.md index 2fde290..8027913 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,19 @@ Pour couper le serveur, on tue le maître UWSGI, kill -INT `cat ~/django-med/uwsgi.pid` ``` +Pour reverse-proxyfier le serveur derrière Apache, on place dans `~/www/.htaccess` : + +```apache +RewriteEngine On +RewriteRule ^(.*)$ unix:/home/c/club-med/django-med/uwsgi.sock|fcgi://localhost [P,NE,QSA,L] +``` + +Pour servir les fichiers statiques, on crée un lien symbolique : + +```bash +ln -s ~/django-med/static ~/www/static +``` + Il est néanmoins une mauvaise idée de faire de la production sur SQLite, on configure donc ensuite Django et une base de données. From 4b553386b0678b8fbda8300581df772d7a5809b8 Mon Sep 17 00:00:00 2001 From: Yohann D'anello Date: Sun, 9 Feb 2020 16:14:47 +0100 Subject: [PATCH 10/79] Add static files to htaccess --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 8027913..b46d37b 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Pour reverse-proxyfier le serveur derrière Apache, on place dans `~/www/.htacce ```apache RewriteEngine On +RewriteCond %{REQUEST_FILENAME} !-f RewriteRule ^(.*)$ unix:/home/c/club-med/django-med/uwsgi.sock|fcgi://localhost [P,NE,QSA,L] ``` From 5623607e4f7db4273e1598693684b34ebd40de8e Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 16:41:43 +0100 Subject: [PATCH 11/79] Add MySQL to requirements --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 31b84ae..2e68d39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,5 @@ djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 psycopg2==2.7.7 -uwsgi==2.0.18 \ No newline at end of file +uwsgi==2.0.18 +mysqlclient==1.4.3 From a5c560307ae09d366320d0c990c81d25e1b03bbd Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 16:59:18 +0100 Subject: [PATCH 12/79] Better way to create DB --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b46d37b..ec8af6d 100644 --- a/README.md +++ b/README.md @@ -89,9 +89,9 @@ ainsi qu'un user med et un mot de passe associé. Voici les étapes à executer pour MySQL : ```SQL -CREATE DATABASE med; -CREATE USER 'med'@'localhost' IDENTIFIED BY 'password'; -GRANT ALL PRIVILEGES ON med.* TO 'med'@'localhost'; +CREATE DATABASE `club-med-cartons` CHARACTER SET utf8; +CREATE USER `club-med-cartons`@`localhost` IDENTIFIED BY 'MY_SUPER_SECRET_PASSWORD' +GRANT ALL PRIVILEGES ON `club-med-cartons`.* TO `club-med-cartons`@`localhost`; FLUSH PRIVILEGES; ``` From 0f0e5fcd2516fadbc0583c820ea263a9af3d359a Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 9 Feb 2020 17:30:13 +0100 Subject: [PATCH 13/79] Blame MySQL --- README.md | 17 ++++------------- med/settings_local.example.py | 15 ++------------- 2 files changed, 6 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index ec8af6d..8ef4bff 100644 --- a/README.md +++ b/README.md @@ -86,21 +86,12 @@ on configure donc ensuite Django et une base de données. Sur le serveur MySQL ou PostgreSQL, il est nécessaire de créer une base de donnée med, ainsi qu'un user med et un mot de passe associé. -Voici les étapes à executer pour MySQL : +Voici les étapes à executer pour PostgreSQL : ```SQL -CREATE DATABASE `club-med-cartons` CHARACTER SET utf8; -CREATE USER `club-med-cartons`@`localhost` IDENTIFIED BY 'MY_SUPER_SECRET_PASSWORD' -GRANT ALL PRIVILEGES ON `club-med-cartons`.* TO `club-med-cartons`@`localhost`; -FLUSH PRIVILEGES; -``` - -Et pour PostgreSQL : - -```SQL -CREATE DATABASE med; -CREATE USER med WITH PASSWORD 'password'; -GRANT ALL PRIVILEGES ON DATABASE med TO med; +CREATE DATABASE "club-med"; +CREATE USER "club-med" WITH PASSWORD 'MY-STRONG-PASSWORD'; +GRANT ALL PRIVILEGES ON DATABASE "club-med" TO "club-med"; ``` ## Exemple de groupes de droits diff --git a/med/settings_local.example.py b/med/settings_local.example.py index f0d61c3..51fb051 100644 --- a/med/settings_local.example.py +++ b/med/settings_local.example.py @@ -33,21 +33,10 @@ DEBUG = True DATABASES = { 'default': { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'med', - 'USER': 'med', + 'NAME': 'club-med', + 'USER': 'club-med', 'PASSWORD': 'password_to_store_in_env', 'HOST': 'db', 'PORT': '', } } - -# or MySQL database for Zamok -# DATABASES = { -# 'default': { -# 'ENGINE': 'django.db.backends.mysql', -# 'NAME': 'club-med', -# 'USER': 'club-med', -# 'PASSWORD': 'CHANGE ME !!!', -# 'HOST': 'localhost', -# }, -# } \ No newline at end of file From 87063e267e5672448ecc36935e4037f2b6a28bf6 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 9 Feb 2020 17:55:51 +0100 Subject: [PATCH 14/79] Remove `Clef` Model --- README.md | 5 ----- users/admin.py | 10 +--------- users/models.py | 25 ------------------------- 3 files changed, 1 insertion(+), 39 deletions(-) diff --git a/README.md b/README.md index 8ef4bff..ed2afdf 100644 --- a/README.md +++ b/README.md @@ -106,10 +106,6 @@ bureau users | Can add adhesion users | Can change adhesion users | Can delete adhesion - users | Can view clef - users | Can add clef - users | Can change clef - users | Can delete clef users | Can view user users | Can add user users | Can change user @@ -134,7 +130,6 @@ keyholder media | Can change borrowed item media | Can delete borrowed item users | Can view user - users | Can view clef users (default group for everyone) media | Can view author diff --git a/users/admin.py b/users/admin.py index aef5357..effd1c0 100644 --- a/users/admin.py +++ b/users/admin.py @@ -13,14 +13,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import UserCreationAdminForm -from .models import Adhesion, Clef, User - - -class ClefAdmin(VersionAdmin): - list_display = ('name', 'owner', 'comment') - ordering = ('name',) - search_fields = ('name', 'owner__username', 'comment') - autocomplete_fields = ('owner',) +from .models import Adhesion, User class AdhesionAdmin(VersionAdmin): @@ -116,4 +109,3 @@ class UserAdmin(VersionAdmin, BaseUserAdmin): admin_site.register(User, UserAdmin) admin_site.register(Adhesion, AdhesionAdmin) -admin_site.register(Clef, ClefAdmin) diff --git a/users/models.py b/users/models.py index 95e916f..e333b58 100644 --- a/users/models.py +++ b/users/models.py @@ -50,31 +50,6 @@ class User(AbstractUser): return last_year and self in last_year.members.all() -class Clef(models.Model): - name = models.CharField( - verbose_name=_('name'), - max_length=255, - unique=True, - ) - owner = models.ForeignKey( - 'User', - on_delete=models.PROTECT, - verbose_name=_('owner'), - blank=True, - null=True, - ) - comment = models.CharField( - verbose_name=_('comment'), - max_length=255, - null=True, - blank=True, - ) - - class Meta: - verbose_name = _('key') - verbose_name_plural = _('keys') - - class Adhesion(models.Model): starting_in = models.IntegerField( verbose_name=_('starting in'), From ac866cc0baf645bffacb26ccacfad30a01344af2 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 9 Feb 2020 18:17:26 +0100 Subject: [PATCH 15/79] Remove `Clef` Model --- users/migrations/0040_delete_clef.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 users/migrations/0040_delete_clef.py diff --git a/users/migrations/0040_delete_clef.py b/users/migrations/0040_delete_clef.py new file mode 100644 index 0000000..dc061cf --- /dev/null +++ b/users/migrations/0040_delete_clef.py @@ -0,0 +1,16 @@ +# Generated by Django 2.2.4 on 2020-02-09 16:54 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('users', '0039_auto_20190810_1610'), + ] + + operations = [ + migrations.DeleteModel( + name='Clef', + ), + ] From 47bf025145f374bf05ba4dcf51b20f316ff20db3 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 9 Feb 2020 18:18:02 +0100 Subject: [PATCH 16/79] IPV6 Barcode scanner --- tool_barcode_getblue.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tool_barcode_getblue.py b/tool_barcode_getblue.py index 0fdaf55..2a24235 100644 --- a/tool_barcode_getblue.py +++ b/tool_barcode_getblue.py @@ -1,6 +1,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer import os +import socket """ GetBlue Android parameters @@ -28,8 +29,12 @@ class Server(BaseHTTPRequestHandler): self._set_headers() +class HTTPServerV6(HTTPServer): + address_family = socket.AF_INET6 + + if __name__ == "__main__": - server_address = ('', 8080) - httpd = HTTPServer(server_address, Server) + server_address = ('::', 8080) + httpd = HTTPServerV6(server_address, Server) print('Starting httpd...') httpd.serve_forever() From a52c9f5cb30310ddb4ddf219757ad994dc392e55 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Sun, 9 Feb 2020 21:47:24 +0100 Subject: [PATCH 17/79] Working htaccess --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ed2afdf..6e08f0c 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,14 @@ Pour reverse-proxyfier le serveur derrière Apache, on place dans `~/www/.htacce ```apache RewriteEngine On + +# UWSGI socket +RewriteRule ^django.wsgi/(.*)$ unix:/home/c/club-med/django-med/uwsgi.sock|fcgi://localhost/ [P,NE,L] + +# When not a file and not starting with django.wsgi, then forward to UWSGI +RewriteCond %{REQUEST_URI} !^/django.wsgi/ RewriteCond %{REQUEST_FILENAME} !-f -RewriteRule ^(.*)$ unix:/home/c/club-med/django-med/uwsgi.sock|fcgi://localhost [P,NE,QSA,L] +RewriteRule ^(.*)$ /django.wsgi/$1 [QSA,L] ``` Pour servir les fichiers statiques, on crée un lien symbolique : From 552d2b8f0e59696882c3d979831b4f7f8eccb45c Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 03:08:08 +0100 Subject: [PATCH 18/79] Required fields are not checked when the ISBN is typed --- media/forms.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index 24b5cbd..e0e29e1 100644 --- a/media/forms.py +++ b/media/forms.py @@ -60,6 +60,8 @@ class MediaAdminForm(ModelForm): """ If user fetch ISBN data, then download data before validating the form """ + super().clean() + # TODO implement authors, side_identifier if "_continue" in self.request.POST: isbn = self.cleaned_data.get('isbn') @@ -70,4 +72,25 @@ class MediaAdminForm(ModelForm): # Try with OpenLibrary self.download_data_openlibrary(isbn) - return super().clean() + return self.cleaned_data + + def _clean_fields(self): + for name, field in self.fields.items(): + # value_from_datadict() gets the data from the data dictionaries. + # Each widget type knows how to retrieve its own data, because some + # widgets split data over several HTML fields. + if field.disabled: + value = self.get_initial_for_field(field, name) + else: + value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name)) + from django.core.exceptions import ValidationError + try: + # We don't want to check a field when we enter an ISBN. + if "_continue" not in self.request.POST or not self.cleaned_data.get('isbn'): + value = field.clean(value) + self.cleaned_data[name] = value + if hasattr(self, 'clean_%s' % name): + value = getattr(self, 'clean_%s' % name)() + self.cleaned_data[name] = value + except ValidationError as e: + self.add_error(name, e) From 6cd7f883b90515cc4a5d55d9ccf0c012ab75ad73 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 03:58:52 +0100 Subject: [PATCH 19/79] Scrap author and illustrator (may not work for some books) --- media/scraper.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/media/scraper.py b/media/scraper.py index ec91d5e..02e8057 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -56,6 +56,8 @@ class BedetequeScraper: regex_subtitle = r'

\s*(.*)

' regex_publish_date = r'datePublished\" content=\"([\d-]*)\">' regex_nb_of_pages = r'numberOfPages\">(\d*)(((?!<).)*)' + regex_illustrator = r'(((?!<).)*)' data = { 'external_url': bd_url, @@ -87,4 +89,16 @@ class BedetequeScraper: if search_nb_pages and search_nb_pages.group(1).isnumeric(): data['number_of_pages'] = search_nb_pages.group(1) + # Get author and illustrator + author = re.search(regex_author, content) + if not 'author' in data: + data['author'] = list() + if author: + data['author'].append(author.group(1)) + illustrator = re.search(regex_illustrator, content) + if illustrator: + data['author'].append(illustrator.group(1)) + + print(data) + return data From 3d62973634da77007c437ccc3605328f2a64f497 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 04:16:07 +0100 Subject: [PATCH 20/79] Generate side identifier --- media/scraper.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/media/scraper.py b/media/scraper.py index 02e8057..931f2c6 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -99,6 +99,12 @@ class BedetequeScraper: if illustrator: data['author'].append(illustrator.group(1)) - print(data) + side_identifier = "{:.3} {:.3}".format(data['author'][0].upper(), data['title'].upper(),) + if data['subtitle']: + start = data['subtitle'].split(' ')[0].replace('.', '') + print("start:", start) + if start.isnumeric(): + side_identifier += " {:0>2}".format(start,) + data['side_identifier'] = side_identifier return data From f08271689504ab6b752146fcf47a95130e309d17 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 04:21:41 +0100 Subject: [PATCH 21/79] The name of an author is the second word (if no ,) --- media/scraper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/media/scraper.py b/media/scraper.py index 931f2c6..7861a0a 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -99,7 +99,10 @@ class BedetequeScraper: if illustrator: data['author'].append(illustrator.group(1)) - side_identifier = "{:.3} {:.3}".format(data['author'][0].upper(), data['title'].upper(),) + author_name = data['author'][0] + if ',' not in author_name: + author_name = author_name.split(' ')[1] + side_identifier = "{:.3} {:.3}".format(author_name.upper(), data['title'].upper(),) if data['subtitle']: start = data['subtitle'].split(' ')[0].replace('.', '') print("start:", start) From f3f9c70de9ad6a70da933c22ec506f09581d58e3 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 04:59:39 +0100 Subject: [PATCH 22/79] Pipelines --- media/forms.py | 6 ++++-- media/scraper.py | 7 ++++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/media/forms.py b/media/forms.py index e0e29e1..25a8d49 100644 --- a/media/forms.py +++ b/media/forms.py @@ -82,11 +82,13 @@ class MediaAdminForm(ModelForm): if field.disabled: value = self.get_initial_for_field(field, name) else: - value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name)) + value = field.widget.value_from_datadict( + self.data, self.files, self.add_prefix(name)) from django.core.exceptions import ValidationError try: # We don't want to check a field when we enter an ISBN. - if "_continue" not in self.request.POST or not self.cleaned_data.get('isbn'): + if "_continue" not in self.request.POST \ + or not self.cleaned_data.get('isbn'): value = field.clean(value) self.cleaned_data[name] = value if hasattr(self, 'clean_%s' % name): diff --git a/media/scraper.py b/media/scraper.py index 7861a0a..94e95ea 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -91,7 +91,7 @@ class BedetequeScraper: # Get author and illustrator author = re.search(regex_author, content) - if not 'author' in data: + if 'author' not in data: data['author'] = list() if author: data['author'].append(author.group(1)) @@ -100,9 +100,10 @@ class BedetequeScraper: data['author'].append(illustrator.group(1)) author_name = data['author'][0] - if ',' not in author_name: + if ',' not in author_name and ' ' in author_name: author_name = author_name.split(' ')[1] - side_identifier = "{:.3} {:.3}".format(author_name.upper(), data['title'].upper(),) + side_identifier = "{:.3} {:.3}".format(author_name.upper(), + data['title'].upper(),) if data['subtitle']: start = data['subtitle'].split(' ')[0].replace('.', '') print("start:", start) From ac8d91ac5e0819545a970b5ca95987064ff3b034 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 05:03:53 +0100 Subject: [PATCH 23/79] Correct test --- media/tests/test_templates.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/tests/test_templates.py b/media/tests/test_templates.py index e4ce582..b9d2e75 100644 --- a/media/tests/test_templates.py +++ b/media/tests/test_templates.py @@ -54,7 +54,7 @@ class TemplateTests(TestCase): 'admin:media_media_change', args=[self.dummy_media1.id], ), data=data) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 302) def test_media_emprunt_changelist(self): response = self.client.get(reverse('admin:media_emprunt_changelist')) From c7d804d9bfe7ca43471129902965669da012bd9a Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 05:06:24 +0100 Subject: [PATCH 24/79] One line was too long --- media/scraper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/scraper.py b/media/scraper.py index 94e95ea..fe86be8 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -57,7 +57,7 @@ class BedetequeScraper: regex_publish_date = r'datePublished\" content=\"([\d-]*)\">' regex_nb_of_pages = r'numberOfPages\">(\d*)(((?!<).)*)' - regex_illustrator = r'(((?!<).)*)' + regex_illustrator = r'span itemprop=\"illustrator\">(((?!<).)*) Date: Mon, 10 Feb 2020 11:29:26 +0100 Subject: [PATCH 25/79] Fix authors & openlibrary scrap --- media/forms.py | 37 ++++++++++++++++++++++++++++++++++++- media/scraper.py | 28 +++++++++------------------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/media/forms.py b/media/forms.py index 25a8d49..8dd1df2 100644 --- a/media/forms.py +++ b/media/forms.py @@ -7,6 +7,7 @@ import urllib.request from django.forms import ModelForm +from .models import Auteur from .scraper import BedetequeScraper @@ -30,6 +31,7 @@ class MediaAdminForm(ModelForm): # If results, then take the most accurate data = scraper.scrap_bd_info(r[0]) self.cleaned_data.update(data) + print(self.cleaned_data) return True def download_data_openlibrary(self, isbn): @@ -53,6 +55,23 @@ class MediaAdminForm(ModelForm): if 'number_of_pages' in data: self.cleaned_data['number_of_pages'] = \ data['number_of_pages'] + elif not self.cleaned_data['number_of_pages']: + self.cleaned_data['number_of_pages'] = 0 + if 'publish_date' in data: + months = ['January', 'February', "March", "April", "Mai", + "June", "July", "August", "September", + "October", "November","December"] + split = data['publish_date'].replace(',', '').split(' ') + self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}"\ + .format(split[2], months.index(split[0])+1, + int(split[1]),) + if 'authors' in data: + if 'author' not in self.cleaned_data: + self.cleaned_data['authors'] = list() + for author in data['authors']: + author_obj = Auteur.objects.get_or_create( + name=author['name'])[0] + self.cleaned_data['authors'].append(author_obj) return True return False @@ -62,7 +81,6 @@ class MediaAdminForm(ModelForm): """ super().clean() - # TODO implement authors, side_identifier if "_continue" in self.request.POST: isbn = self.cleaned_data.get('isbn') if isbn: @@ -72,6 +90,23 @@ class MediaAdminForm(ModelForm): # Try with OpenLibrary self.download_data_openlibrary(isbn) + if self.cleaned_data['authors']: + author_name = self.cleaned_data['authors'][0].name + if ',' not in author_name and ' ' in author_name: + author_name = author_name.split(' ')[1] + side_identifier = "{:.3} {:.3}".format( + author_name.upper(), + self.cleaned_data['title'].upper(), ) + + if self.cleaned_data['subtitle']: + start = self.cleaned_data['subtitle'] \ + .split(' ')[0].replace('.', '') + + if start.isnumeric(): + side_identifier += " {:0>2}".format(start, ) + + self.cleaned_data['side_identifier'] = side_identifier + return self.cleaned_data def _clean_fields(self): diff --git a/media/scraper.py b/media/scraper.py index fe86be8..2aaf670 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -5,6 +5,8 @@ import re import requests +from media.models import Auteur + class BedetequeScraper: """ @@ -75,10 +77,6 @@ class BedetequeScraper: subtitle = subtitle.replace('', '') data['subtitle'] = ' '.join(subtitle.split()) - # TODO implement author - # regex_author = r'author\">([^<]*) Date: Mon, 10 Feb 2020 11:47:59 +0100 Subject: [PATCH 26/79] Clean code --- media/forms.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/media/forms.py b/media/forms.py index 8dd1df2..c16cb92 100644 --- a/media/forms.py +++ b/media/forms.py @@ -43,6 +43,7 @@ class MediaAdminForm(ModelForm): "&format=json&jscmd=data".format(isbn) with urllib.request.urlopen(api_url) as url: data = json.loads(url.read().decode()) + if data and data['ISBN:' + isbn]: data = data['ISBN:' + isbn] if 'url' in data: @@ -52,19 +53,22 @@ class MediaAdminForm(ModelForm): self.cleaned_data['title'] = data['title'] if 'subtitle' in data: self.cleaned_data['subtitle'] = data['subtitle'] + if 'number_of_pages' in data: self.cleaned_data['number_of_pages'] = \ data['number_of_pages'] elif not self.cleaned_data['number_of_pages']: self.cleaned_data['number_of_pages'] = 0 + if 'publish_date' in data: months = ['January', 'February', "March", "April", "Mai", "June", "July", "August", "September", - "October", "November","December"] + "October", "November", "December"] split = data['publish_date'].replace(',', '').split(' ') self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}"\ - .format(split[2], months.index(split[0])+1, + .format(split[2], months.index(split[0]) + 1, int(split[1]),) + if 'authors' in data: if 'author' not in self.cleaned_data: self.cleaned_data['authors'] = list() @@ -99,8 +103,8 @@ class MediaAdminForm(ModelForm): self.cleaned_data['title'].upper(), ) if self.cleaned_data['subtitle']: - start = self.cleaned_data['subtitle'] \ - .split(' ')[0].replace('.', '') + start = self.cleaned_data['subtitle'].split(' ')[0] \ + .replace('.', '') if start.isnumeric(): side_identifier += " {:0>2}".format(start, ) From 778e3239a452a108a9c540a251cde2406232f081 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 12:03:40 +0100 Subject: [PATCH 27/79] Less complexity for download_data_openlibrary --- entrypoint.sh | 28 +++++++++++++++------------- requirements.txt | 2 +- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index aa2d908..96fcc64 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,19 +11,21 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input +python manage.py runserver 0.0.0.0:8000 + # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped -uwsgi --chdir="$(pwd)" \ - --module=med.wsgi:application \ - --env DJANGO_SETTINGS_MODULE=med.settings \ - --master \ - --pidfile="$(pwd)/uwsgi.pid" \ - --socket="$(pwd)/uwsgi.sock" \ - --processes=5 \ - --chmod-socket=600 \ - --harakiri=20 \ - --max-requests=5000 \ - --vacuum \ - --daemonize="$(pwd)/uwsgi.log" \ - --protocol=fastcgi \ No newline at end of file +#uwsgi --chdir="$(pwd)" \ +# --module=med.wsgi:application \ +# --env DJANGO_SETTINGS_MODULE=med.settings \ +# --master \ +# --pidfile="$(pwd)/uwsgi.pid" \ +# --socket="$(pwd)/uwsgi.sock" \ +# --processes=5 \ +# --chmod-socket=600 \ +# --harakiri=20 \ +# --max-requests=5000 \ +# --vacuum \ +# --daemonize="$(pwd)/uwsgi.log" \ +# --protocol=fastcgi \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2e68d39..f02f58e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2==2.7.7 +psycopg2 uwsgi==2.0.18 mysqlclient==1.4.3 From 939efe01a08c02f1831fa46c167a75cb08c93905 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 12:08:48 +0100 Subject: [PATCH 28/79] Less complexity for download_data_openlibrary --- entrypoint.sh | 28 +++++++++++------------ media/forms.py | 59 +++++++++++++++++++++++++----------------------- requirements.txt | 2 +- 3 files changed, 45 insertions(+), 44 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 96fcc64..aa2d908 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,21 +11,19 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input -python manage.py runserver 0.0.0.0:8000 - # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped -#uwsgi --chdir="$(pwd)" \ -# --module=med.wsgi:application \ -# --env DJANGO_SETTINGS_MODULE=med.settings \ -# --master \ -# --pidfile="$(pwd)/uwsgi.pid" \ -# --socket="$(pwd)/uwsgi.sock" \ -# --processes=5 \ -# --chmod-socket=600 \ -# --harakiri=20 \ -# --max-requests=5000 \ -# --vacuum \ -# --daemonize="$(pwd)/uwsgi.log" \ -# --protocol=fastcgi \ No newline at end of file +uwsgi --chdir="$(pwd)" \ + --module=med.wsgi:application \ + --env DJANGO_SETTINGS_MODULE=med.settings \ + --master \ + --pidfile="$(pwd)/uwsgi.pid" \ + --socket="$(pwd)/uwsgi.sock" \ + --processes=5 \ + --chmod-socket=600 \ + --harakiri=20 \ + --max-requests=5000 \ + --vacuum \ + --daemonize="$(pwd)/uwsgi.log" \ + --protocol=fastcgi \ No newline at end of file diff --git a/media/forms.py b/media/forms.py index c16cb92..d077219 100644 --- a/media/forms.py +++ b/media/forms.py @@ -48,37 +48,40 @@ class MediaAdminForm(ModelForm): data = data['ISBN:' + isbn] if 'url' in data: # Fill the data - self.cleaned_data['external_url'] = data['url'] - if 'title' in data: - self.cleaned_data['title'] = data['title'] - if 'subtitle' in data: - self.cleaned_data['subtitle'] = data['subtitle'] - - if 'number_of_pages' in data: - self.cleaned_data['number_of_pages'] = \ - data['number_of_pages'] - elif not self.cleaned_data['number_of_pages']: - self.cleaned_data['number_of_pages'] = 0 - - if 'publish_date' in data: - months = ['January', 'February', "March", "April", "Mai", - "June", "July", "August", "September", - "October", "November", "December"] - split = data['publish_date'].replace(',', '').split(' ') - self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}"\ - .format(split[2], months.index(split[0]) + 1, - int(split[1]),) - - if 'authors' in data: - if 'author' not in self.cleaned_data: - self.cleaned_data['authors'] = list() - for author in data['authors']: - author_obj = Auteur.objects.get_or_create( - name=author['name'])[0] - self.cleaned_data['authors'].append(author_obj) + self.parse_data_openlibrary(data) return True return False + def parse_data_openlibrary(self, data): + self.cleaned_data['external_url'] = data['url'] + if 'title' in data: + self.cleaned_data['title'] = data['title'] + if 'subtitle' in data: + self.cleaned_data['subtitle'] = data['subtitle'] + + if 'number_of_pages' in data: + self.cleaned_data['number_of_pages'] = \ + data['number_of_pages'] + elif not self.cleaned_data['number_of_pages']: + self.cleaned_data['number_of_pages'] = 0 + + if 'publish_date' in data: + months = ['January', 'February', "March", "April", "Mai", + "June", "July", "August", "September", + "October", "November", "December"] + split = data['publish_date'].replace(',', '').split(' ') + self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ + .format(split[2], months.index(split[0]) + 1, + int(split[1]), ) + + if 'authors' in data: + if 'author' not in self.cleaned_data: + self.cleaned_data['authors'] = list() + for author in data['authors']: + author_obj = Auteur.objects.get_or_create( + name=author['name'])[0] + self.cleaned_data['authors'].append(author_obj) + def clean(self): """ If user fetch ISBN data, then download data before validating the form diff --git a/requirements.txt b/requirements.txt index f02f58e..2e68d39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2 +psycopg2==2.7.7 uwsgi==2.0.18 mysqlclient==1.4.3 From e4d1ed852f660a448b5465d00ca1f44e181d3b7f Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 16:32:13 +0100 Subject: [PATCH 29/79] Fix page numbers --- media/scraper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/media/scraper.py b/media/scraper.py index 2aaf670..045a8b0 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -86,6 +86,8 @@ class BedetequeScraper: search_nb_pages = re.search(regex_nb_of_pages, content) if search_nb_pages and search_nb_pages.group(1).isnumeric(): data['number_of_pages'] = search_nb_pages.group(1) + elif 'number_of_pages' not in data: + data['number_of_pages'] = 0 # Get author and illustrator author = re.search(regex_author, content) From dc23ac039692724a55186837f3bc754cd041d11c Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 16:47:05 +0100 Subject: [PATCH 30/79] Remove special chars from side identifiers --- media/forms.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/media/forms.py b/media/forms.py index d077219..a9fe699 100644 --- a/media/forms.py +++ b/media/forms.py @@ -4,6 +4,7 @@ import json import urllib.request +import unicodedata from django.forms import ModelForm @@ -102,8 +103,8 @@ class MediaAdminForm(ModelForm): if ',' not in author_name and ' ' in author_name: author_name = author_name.split(' ')[1] side_identifier = "{:.3} {:.3}".format( - author_name.upper(), - self.cleaned_data['title'].upper(), ) + author_name, + self.cleaned_data['title'], ) if self.cleaned_data['subtitle']: start = self.cleaned_data['subtitle'].split(' ')[0] \ @@ -112,6 +113,13 @@ class MediaAdminForm(ModelForm): if start.isnumeric(): side_identifier += " {:0>2}".format(start, ) + # Normalize side identifier, in order to remove accents + side_identifier = ''.join( + char + for char in unicodedata.normalize('NFKD', side_identifier.casefold()) + if all(not unicodedata.category(char).startswith(cat) + for cat in {'M', 'P', 'Z', 'C'}) or char == ' ' + ).casefold().upper() self.cleaned_data['side_identifier'] = side_identifier return self.cleaned_data From 8dbf0494c2fabedc252741b2ef2a31aaba10ed83 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 16:48:57 +0100 Subject: [PATCH 31/79] The last name of the author is considered for the side identifier --- media/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index a9fe699..7586c5e 100644 --- a/media/forms.py +++ b/media/forms.py @@ -101,7 +101,7 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['authors']: author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: - author_name = author_name.split(' ')[1] + author_name = author_name.split(' ')[-1] side_identifier = "{:.3} {:.3}".format( author_name, self.cleaned_data['title'], ) From 11f0eff4d4d3cfda785a292e7bd00c25069d27c9 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:07:06 +0100 Subject: [PATCH 32/79] Clean code --- media/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index 7586c5e..fd0418b 100644 --- a/media/forms.py +++ b/media/forms.py @@ -116,7 +116,8 @@ class MediaAdminForm(ModelForm): # Normalize side identifier, in order to remove accents side_identifier = ''.join( char - for char in unicodedata.normalize('NFKD', side_identifier.casefold()) + for char in unicodedata.normalize( + 'NFKD', side_identifier.casefold()) if all(not unicodedata.category(char).startswith(cat) for cat in {'M', 'P', 'Z', 'C'}) or char == ' ' ).casefold().upper() From ea821483d0a3fdaf825facc5a0646966a7daaf6d Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:27:30 +0100 Subject: [PATCH 33/79] Add author note --- media/forms.py | 4 ++ media/locale/fr/LC_MESSAGES/django.po | 54 ++++++++++++++------------- media/migrations/0025_auteur_note.py | 18 +++++++++ media/models.py | 5 +++ 4 files changed, 56 insertions(+), 25 deletions(-) create mode 100644 media/migrations/0025_auteur_note.py diff --git a/media/forms.py b/media/forms.py index fd0418b..e3a7589 100644 --- a/media/forms.py +++ b/media/forms.py @@ -99,6 +99,10 @@ class MediaAdminForm(ModelForm): self.download_data_openlibrary(isbn) if self.cleaned_data['authors']: + authors: list = self.cleaned_data['authors'] + def sort(author): + return str(-author.note) + "." + author.name + authors.sort(key=sort) author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: author_name = author_name.split(' ')[-1] diff --git a/media/locale/fr/LC_MESSAGES/django.po b/media/locale/fr/LC_MESSAGES/django.po index d8ceca6..5bbae5c 100644 --- a/media/locale/fr/LC_MESSAGES/django.po +++ b/media/locale/fr/LC_MESSAGES/django.po @@ -41,103 +41,107 @@ msgstr "nom" msgid "author" msgstr "auteur" -#: models.py:30 +#: models.py:23 +msgid "note" +msgstr "note" + +#: models.py:35 msgid "ISBN" msgstr "ISBN" -#: models.py:31 +#: models.py:36 msgid "You may be able to scan it from a bar code." msgstr "Peut souvent être scanné à partir du code barre." -#: models.py:36 +#: models.py:41 msgid "title" msgstr "titre" -#: models.py:40 +#: models.py:45 msgid "subtitle" msgstr "sous-titre" -#: models.py:46 +#: models.py:51 msgid "external URL" msgstr "URL externe" -#: models.py:51 +#: models.py:56 msgid "side identifier" msgstr "côte" -#: models.py:59 +#: models.py:64 msgid "number of pages" msgstr "nombre de pages" -#: models.py:64 +#: models.py:69 msgid "publish date" msgstr "date de publication" -#: models.py:76 +#: models.py:81 msgid "medium" msgstr "medium" -#: models.py:77 +#: models.py:82 msgid "media" msgstr "media" -#: models.py:89 +#: models.py:94 msgid "borrower" msgstr "emprunteur" -#: models.py:92 +#: models.py:97 msgid "borrowed on" msgstr "emprunté le" -#: models.py:97 +#: models.py:102 msgid "given back on" msgstr "rendu le" -#: models.py:103 +#: models.py:108 msgid "borrowed with" msgstr "emprunté avec" -#: models.py:104 +#: models.py:109 msgid "The keyholder that registered this borrowed item." msgstr "Le permanencier qui enregistre cet emprunt." -#: models.py:113 +#: models.py:118 msgid "The keyholder to whom this item was given back." msgstr "Le permanencier à qui l'emprunt a été rendu." -#: models.py:120 +#: models.py:125 msgid "borrowed item" msgstr "emprunt" -#: models.py:121 +#: models.py:126 msgid "borrowed items" msgstr "emprunts" -#: models.py:141 +#: models.py:146 msgid "owner" msgstr "propriétaire" -#: models.py:146 +#: models.py:151 msgid "duration" msgstr "durée" -#: models.py:150 +#: models.py:155 msgid "minimum number of players" msgstr "nombre minimum de joueurs" -#: models.py:154 +#: models.py:159 msgid "maximum number of players" msgstr "nombre maximum de joueurs" -#: models.py:160 +#: models.py:165 msgid "comment" msgstr "commentaire" -#: models.py:167 +#: models.py:172 msgid "game" msgstr "jeu" -#: models.py:168 +#: models.py:173 msgid "games" msgstr "jeux" diff --git a/media/migrations/0025_auteur_note.py b/media/migrations/0025_auteur_note.py new file mode 100644 index 0000000..3639431 --- /dev/null +++ b/media/migrations/0025_auteur_note.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.4 on 2020-02-10 16:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0024_auto_20190816_1356'), + ] + + operations = [ + migrations.AddField( + model_name='auteur', + name='note', + field=models.IntegerField(default=0, verbose_name='note'), + ), + ] diff --git a/media/models.py b/media/models.py index 533ccc5..bad1c1d 100644 --- a/media/models.py +++ b/media/models.py @@ -16,6 +16,11 @@ class Auteur(models.Model): verbose_name=_('name'), ) + note = models.IntegerField( + default=0, + verbose_name=_("note"), + ) + def __str__(self): return self.name From 70045d4e2d13b9d40040ba0719b149be51267479 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:29:10 +0100 Subject: [PATCH 34/79] Python 3.5 support --- media/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index e3a7589..7bd1056 100644 --- a/media/forms.py +++ b/media/forms.py @@ -99,7 +99,7 @@ class MediaAdminForm(ModelForm): self.download_data_openlibrary(isbn) if self.cleaned_data['authors']: - authors: list = self.cleaned_data['authors'] + authors = self.cleaned_data['authors'] def sort(author): return str(-author.note) + "." + author.name authors.sort(key=sort) From 71a8aa065ba1409acd3ba8c5b7a353123cac4649 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:35:07 +0100 Subject: [PATCH 35/79] Fix author sort --- media/forms.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index 7bd1056..18f91a0 100644 --- a/media/forms.py +++ b/media/forms.py @@ -100,8 +100,9 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] + old_authors = authors.copy() def sort(author): - return str(-author.note) + "." + author.name + return str(-author.note) + "." + str(old_authors.index(author)) + "." + author.name authors.sort(key=sort) author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: From ef710bf96489dc18f008cb41eba01036b4559d79 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:35:35 +0100 Subject: [PATCH 36/79] Fix author sort --- media/forms.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/media/forms.py b/media/forms.py index 18f91a0..b38e091 100644 --- a/media/forms.py +++ b/media/forms.py @@ -102,7 +102,9 @@ class MediaAdminForm(ModelForm): authors = self.cleaned_data['authors'] old_authors = authors.copy() def sort(author): - return str(-author.note) + "." + str(old_authors.index(author)) + "." + author.name + return str(-author.note) + "." \ + + str(old_authors.index(author)) \ + + "." + author.name authors.sort(key=sort) author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: From 53ea1288c103846eadadc03bfab790acccfaa6c6 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:41:05 +0100 Subject: [PATCH 37/79] ISBN is unique --- media/migrations/0026_auto_20200210_1740.py | 20 ++++++++++++++++++++ media/models.py | 1 + 2 files changed, 21 insertions(+) create mode 100644 media/migrations/0026_auto_20200210_1740.py diff --git a/media/migrations/0026_auto_20200210_1740.py b/media/migrations/0026_auto_20200210_1740.py new file mode 100644 index 0000000..1a89523 --- /dev/null +++ b/media/migrations/0026_auto_20200210_1740.py @@ -0,0 +1,20 @@ +# Generated by Django 2.2.4 on 2020-02-10 16:40 + +from django.db import migrations +import media.fields +import media.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0025_auteur_note'), + ] + + operations = [ + migrations.AlterField( + model_name='media', + name='isbn', + field=media.fields.ISBNField(blank=True, help_text='You may be able to scan it from a bar code.', max_length=28, null=True, unique=True, validators=[media.validators.isbn_validator], verbose_name='ISBN'), + ), + ] diff --git a/media/models.py b/media/models.py index bad1c1d..dbfaf04 100644 --- a/media/models.py +++ b/media/models.py @@ -34,6 +34,7 @@ class Media(models.Model): isbn = ISBNField( _('ISBN'), help_text=_('You may be able to scan it from a bar code.'), + unique=True, blank=True, null=True, ) From 3d81977dbd3e69caae28546572446d2dfeb654be Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:51:09 +0100 Subject: [PATCH 38/79] Side identifiers cannot begin with le/la/les/l'/the --- entrypoint.sh | 28 +++++++++++++++------------- media/forms.py | 9 ++++++++- requirements.txt | 2 +- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index aa2d908..96fcc64 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,19 +11,21 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input +python manage.py runserver 0.0.0.0:8000 + # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped -uwsgi --chdir="$(pwd)" \ - --module=med.wsgi:application \ - --env DJANGO_SETTINGS_MODULE=med.settings \ - --master \ - --pidfile="$(pwd)/uwsgi.pid" \ - --socket="$(pwd)/uwsgi.sock" \ - --processes=5 \ - --chmod-socket=600 \ - --harakiri=20 \ - --max-requests=5000 \ - --vacuum \ - --daemonize="$(pwd)/uwsgi.log" \ - --protocol=fastcgi \ No newline at end of file +#uwsgi --chdir="$(pwd)" \ +# --module=med.wsgi:application \ +# --env DJANGO_SETTINGS_MODULE=med.settings \ +# --master \ +# --pidfile="$(pwd)/uwsgi.pid" \ +# --socket="$(pwd)/uwsgi.sock" \ +# --processes=5 \ +# --chmod-socket=600 \ +# --harakiri=20 \ +# --max-requests=5000 \ +# --vacuum \ +# --daemonize="$(pwd)/uwsgi.log" \ +# --protocol=fastcgi \ No newline at end of file diff --git a/media/forms.py b/media/forms.py index b38e091..b1a8f40 100644 --- a/media/forms.py +++ b/media/forms.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later import json +import re import urllib.request import unicodedata @@ -109,9 +110,15 @@ class MediaAdminForm(ModelForm): author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: author_name = author_name.split(' ')[-1] + title_normalized = self.cleaned_data['title'].upper() + title_normalized = re.sub(r'^LE ', '', title_normalized) + title_normalized = re.sub(r'^LA ', '', title_normalized) + title_normalized = re.sub(r'^LES ', '', title_normalized) + title_normalized = re.sub(r'^L\'', '', title_normalized) + title_normalized = re.sub(r'^THE ', '', title_normalized) side_identifier = "{:.3} {:.3}".format( author_name, - self.cleaned_data['title'], ) + title_normalized.replace(' ', ''), ) if self.cleaned_data['subtitle']: start = self.cleaned_data['subtitle'].split(' ')[0] \ diff --git a/requirements.txt b/requirements.txt index 2e68d39..f02f58e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2==2.7.7 +psycopg2 uwsgi==2.0.18 mysqlclient==1.4.3 From 5b2dd84115208f9f2d6a93ba8bd4da15a5fde3ac Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 17:51:49 +0100 Subject: [PATCH 39/79] Docker --- entrypoint.sh | 28 +++++++++++++--------------- requirements.txt | 2 +- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index 96fcc64..aa2d908 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -11,21 +11,19 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input -python manage.py runserver 0.0.0.0:8000 - # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped -#uwsgi --chdir="$(pwd)" \ -# --module=med.wsgi:application \ -# --env DJANGO_SETTINGS_MODULE=med.settings \ -# --master \ -# --pidfile="$(pwd)/uwsgi.pid" \ -# --socket="$(pwd)/uwsgi.sock" \ -# --processes=5 \ -# --chmod-socket=600 \ -# --harakiri=20 \ -# --max-requests=5000 \ -# --vacuum \ -# --daemonize="$(pwd)/uwsgi.log" \ -# --protocol=fastcgi \ No newline at end of file +uwsgi --chdir="$(pwd)" \ + --module=med.wsgi:application \ + --env DJANGO_SETTINGS_MODULE=med.settings \ + --master \ + --pidfile="$(pwd)/uwsgi.pid" \ + --socket="$(pwd)/uwsgi.sock" \ + --processes=5 \ + --chmod-socket=600 \ + --harakiri=20 \ + --max-requests=5000 \ + --vacuum \ + --daemonize="$(pwd)/uwsgi.log" \ + --protocol=fastcgi \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f02f58e..2e68d39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,6 +9,6 @@ python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2 +psycopg2==2.7.7 uwsgi==2.0.18 mysqlclient==1.4.3 From 0548e34568575e6f60f33edb3ad3a873f29f2a5f Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 18:16:19 +0100 Subject: [PATCH 40/79] Subtitles don't need fields --- media/forms.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/media/forms.py b/media/forms.py index b1a8f40..a40ffad 100644 --- a/media/forms.py +++ b/media/forms.py @@ -121,6 +121,10 @@ class MediaAdminForm(ModelForm): title_normalized.replace(' ', ''), ) if self.cleaned_data['subtitle']: + self.cleaned_data['subtitle'] = re.sub(r'', + '', self.cleaned_data['subtitle']) + self.cleaned_data['subtitle'] = re.sub(r'', + '', self.cleaned_data['subtitle']) start = self.cleaned_data['subtitle'].split(' ')[0] \ .replace('.', '') From 47ce447aad9c37627f9ed92c18bf920f625f129c Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 18:18:42 +0100 Subject: [PATCH 41/79] =?UTF-8?q?=C5=92=20=3D>=20OE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media/forms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/media/forms.py b/media/forms.py index a40ffad..ec46a29 100644 --- a/media/forms.py +++ b/media/forms.py @@ -116,6 +116,7 @@ class MediaAdminForm(ModelForm): title_normalized = re.sub(r'^LES ', '', title_normalized) title_normalized = re.sub(r'^L\'', '', title_normalized) title_normalized = re.sub(r'^THE ', '', title_normalized) + title_normalized = re.sub(r'Œ', 'OE', title_normalized) side_identifier = "{:.3} {:.3}".format( author_name, title_normalized.replace(' ', ''), ) From 92dc21f014448628dc6f815094984c41a65c1f8e Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 18:23:13 +0100 Subject: [PATCH 42/79] Remove `(AUT)` fields from titles --- media/forms.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/media/forms.py b/media/forms.py index ec46a29..393024f 100644 --- a/media/forms.py +++ b/media/forms.py @@ -99,6 +99,10 @@ class MediaAdminForm(ModelForm): # Try with OpenLibrary self.download_data_openlibrary(isbn) + if self.cleaned_data['title']: + self.cleaned_data['title'] = re.sub('\(AUT\) ', + '', self.cleaned_data['title']) + if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] old_authors = authors.copy() From 343ab028740465ed287c05d85d6f6aee1f96001f Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 19:33:34 +0100 Subject: [PATCH 43/79] Fixed publish date when there is only the year --- media/forms.py | 9 ++++++--- tool_barcode_getblue.py | 3 +++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/media/forms.py b/media/forms.py index 393024f..9aaf0bf 100644 --- a/media/forms.py +++ b/media/forms.py @@ -72,9 +72,12 @@ class MediaAdminForm(ModelForm): "June", "July", "August", "September", "October", "November", "December"] split = data['publish_date'].replace(',', '').split(' ') - self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ - .format(split[2], months.index(split[0]) + 1, - int(split[1]), ) + if len(split) == 1: + self.cleaned_data['publish_date'] = split[0] + "-01-01" + else: + self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ + .format(split[2], months.index(split[0]) + 1, + int(split[1]), ) if 'authors' in data: if 'author' not in self.cleaned_data: diff --git a/tool_barcode_getblue.py b/tool_barcode_getblue.py index 2a24235..fd5b8cf 100644 --- a/tool_barcode_getblue.py +++ b/tool_barcode_getblue.py @@ -2,6 +2,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer import os import socket +from time import sleep """ GetBlue Android parameters @@ -24,6 +25,8 @@ class Server(BaseHTTPRequestHandler): print("Hey j'ai un ISBN :", isbn) os.system("xdotool type " + isbn) os.system("xdotool key KP_Enter") + sleep(1) + os.system("xdotool click 1") def do_HEAD(self): self._set_headers() From 2753f700a6fa9421df0a8f49d5aa312a16efff57 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 21:12:12 +0100 Subject: [PATCH 44/79] FUCK LINTERS TEST --- media/forms.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/media/forms.py b/media/forms.py index 9aaf0bf..926a485 100644 --- a/media/forms.py +++ b/media/forms.py @@ -4,8 +4,8 @@ import json import re -import urllib.request import unicodedata +import urllib.request from django.forms import ModelForm @@ -103,16 +103,18 @@ class MediaAdminForm(ModelForm): self.download_data_openlibrary(isbn) if self.cleaned_data['title']: - self.cleaned_data['title'] = re.sub('\(AUT\) ', - '', self.cleaned_data['title']) + self.cleaned_data['title'] = re.sub(r'\(AUT\) ', + '', self.cleaned_data['title']) if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] old_authors = authors.copy() + def sort(author): return str(-author.note) + "." \ + str(old_authors.index(author)) \ + "." + author.name + authors.sort(key=sort) author_name = self.cleaned_data['authors'][0].name if ',' not in author_name and ' ' in author_name: @@ -130,9 +132,9 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['subtitle']: self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) start = self.cleaned_data['subtitle'].split(' ')[0] \ .replace('.', '') From ff224d20cd03f1db10a27fb2e8da8aa3af657b99 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 21:21:32 +0100 Subject: [PATCH 45/79] FUCK LINTERS TEST (ERDNAXE C'EST RELOUUU) --- media/forms.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/media/forms.py b/media/forms.py index 926a485..409b33e 100644 --- a/media/forms.py +++ b/media/forms.py @@ -104,7 +104,7 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['title']: self.cleaned_data['title'] = re.sub(r'\(AUT\) ', - '', self.cleaned_data['title']) + '', self.cleaned_data['title']) if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] @@ -112,8 +112,8 @@ class MediaAdminForm(ModelForm): def sort(author): return str(-author.note) + "." \ - + str(old_authors.index(author)) \ - + "." + author.name + + str(old_authors.index(author)) \ + + "." + author.name authors.sort(key=sort) author_name = self.cleaned_data['authors'][0].name @@ -132,9 +132,9 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['subtitle']: self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) start = self.cleaned_data['subtitle'].split(' ')[0] \ .replace('.', '') From b69ded4115070b782ac13bec49f20afb7c8b0506 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Mon, 10 Feb 2020 21:30:23 +0100 Subject: [PATCH 46/79] FUCK LINTERS TEST, FUCK PEP 8 --- media/forms.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/media/forms.py b/media/forms.py index 409b33e..673d513 100644 --- a/media/forms.py +++ b/media/forms.py @@ -104,7 +104,7 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['title']: self.cleaned_data['title'] = re.sub(r'\(AUT\) ', - '', self.cleaned_data['title']) + '', self.cleaned_data['title']) if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] @@ -132,11 +132,11 @@ class MediaAdminForm(ModelForm): if self.cleaned_data['subtitle']: self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + '', self.cleaned_data['subtitle']) start = self.cleaned_data['subtitle'].split(' ')[0] \ - .replace('.', '') + .replace('.', '') if start.isnumeric(): side_identifier += " {:0>2}".format(start, ) From 5b8678188163716722c04f128af0d99ec9cf27a0 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 11 Feb 2020 21:12:00 +0100 Subject: [PATCH 47/79] Fix indentation --- media/forms.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/media/forms.py b/media/forms.py index 673d513..b3c3775 100644 --- a/media/forms.py +++ b/media/forms.py @@ -103,8 +103,11 @@ class MediaAdminForm(ModelForm): self.download_data_openlibrary(isbn) if self.cleaned_data['title']: - self.cleaned_data['title'] = re.sub(r'\(AUT\) ', - '', self.cleaned_data['title']) + self.cleaned_data['title'] = re.sub( + r'\(AUT\) ', + '', + self.cleaned_data['title'] + ) if self.cleaned_data['authors']: authors = self.cleaned_data['authors'] @@ -131,12 +134,18 @@ class MediaAdminForm(ModelForm): title_normalized.replace(' ', ''), ) if self.cleaned_data['subtitle']: - self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) - self.cleaned_data['subtitle'] = re.sub(r'', - '', self.cleaned_data['subtitle']) + self.cleaned_data['subtitle'] = re.sub( + r'', + '', + self.cleaned_data['subtitle'] + ) + self.cleaned_data['subtitle'] = re.sub( + r'', + '', + self.cleaned_data['subtitle'] + ) start = self.cleaned_data['subtitle'].split(' ')[0] \ - .replace('.', '') + .replace('.', '') if start.isnumeric(): side_identifier += " {:0>2}".format(start, ) From 698ae42c9db87e68fd721913930ceb63c6b3a18e Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 11 Feb 2020 21:17:41 +0100 Subject: [PATCH 48/79] Drop old login algorithm --- med/login.py | 84 ------------------------------------------------- med/settings.py | 8 ----- 2 files changed, 92 deletions(-) delete mode 100644 med/login.py diff --git a/med/login.py b/med/login.py deleted file mode 100644 index f4acacd..0000000 --- a/med/login.py +++ /dev/null @@ -1,84 +0,0 @@ -# -*- mode: python; coding: utf-8 -*- -# Copyright (C) 2017-2019 by BDE ENS Paris-Saclay -# SPDX-License-Identifier: GPL-3.0-or-later - - -import binascii -import hashlib -import os -from base64 import decodestring -from base64 import encodestring -from collections import OrderedDict - -from django.contrib.auth import hashers - -ALGO_NAME = "{SSHA}" -ALGO_LEN = len(ALGO_NAME + "$") -DIGEST_LEN = 20 - - -def make_secret(password): - salt = os.urandom(4) - h = hashlib.sha1(password.encode()) - h.update(salt) - return ALGO_NAME + "$" + encodestring(h.digest() + salt).decode()[:-1] - - -def check_password(challenge_password, password): - challenge_bytes = decodestring(challenge_password[ALGO_LEN:].encode()) - digest = challenge_bytes[:DIGEST_LEN] - salt = challenge_bytes[DIGEST_LEN:] - hr = hashlib.sha1(password.encode()) - hr.update(salt) - valid_password = True - # La comparaison est volontairement en temps constant - # (pour éviter les timing-attacks) - for i, j in zip(digest, hr.digest()): - valid_password &= i == j - return valid_password - - -class SSHAPasswordHasher(hashers.BasePasswordHasher): - """ - SSHA password hashing to allow for LDAP auth compatibility - """ - - algorithm = ALGO_NAME - - def encode(self, password, salt, iterations=None): - """ - Hash and salt the given password using SSHA algorithm - - salt is overridden - """ - assert password is not None - return make_secret(password) - - def verify(self, password, encoded): - """ - Check password against encoded using SSHA algorithm - """ - assert encoded.startswith(self.algorithm) - return check_password(encoded, password) - - def safe_summary(self, encoded): - """ - Provides a safe summary ofthe password - """ - assert encoded.startswith(self.algorithm) - hash = encoded[ALGO_LEN:] - hash = binascii.hexlify(decodestring(hash.encode())).decode() - return OrderedDict([ - ('algorithm', self.algorithm), - ('iterations', 0), - ('salt', hashers.mask_hash(hash[2 * DIGEST_LEN:], show=2)), - ('hash', hashers.mask_hash(hash[:2 * DIGEST_LEN])), - ]) - - def harden_runtime(self, password, encoded): - """ - Method implemented to shut up BasePasswordHasher warning - - As we are not using multiple iterations the method is pretty useless - """ - pass diff --git a/med/settings.py b/med/settings.py index 3f24651..b9b08cd 100644 --- a/med/settings.py +++ b/med/settings.py @@ -162,14 +162,6 @@ REST_FRAMEWORK = { # Med configuration PAGINATION_NUMBER = 25 -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.PBKDF2PasswordHasher', - 'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher', - 'django.contrib.auth.hashers.Argon2PasswordHasher', - 'django.contrib.auth.hashers.BCryptSHA256PasswordHasher', - 'med.login.SSHAPasswordHasher', -] - AUTH_USER_MODEL = 'users.User' MAX_EMPRUNT = 5 # Max emprunts From 1b848eede9199b5ecb99a83023945b35dac53a2d Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 11 Feb 2020 21:25:30 +0100 Subject: [PATCH 49/79] Remove redoc --- med/urls.py | 5 ----- theme/templates/admin/base_site.html | 2 +- theme/templates/redoc.html | 12 ------------ 3 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 theme/templates/redoc.html diff --git a/med/urls.py b/med/urls.py index 6cf18ff..51e3f3d 100644 --- a/med/urls.py +++ b/med/urls.py @@ -7,7 +7,6 @@ from django.contrib.auth.views import PasswordResetView from django.urls import include, path from django.views.generic import RedirectView, TemplateView from rest_framework import routers -from rest_framework.schemas import get_schema_view import media.views import users.views @@ -33,10 +32,6 @@ urlpatterns = [ # REST API path('api/', include(router.urls)), path('api-auth/', include('rest_framework.urls')), - path('openapi', login_required(get_schema_view()), name='openapi-schema'), - path('redoc/', - login_required(TemplateView.as_view(template_name='redoc.html')), - name='redoc'), # Include Django Contrib and Core routers path('accounts/password_reset/', PasswordResetView.as_view(), diff --git a/theme/templates/admin/base_site.html b/theme/templates/admin/base_site.html index aebccad..2082f73 100644 --- a/theme/templates/admin/base_site.html +++ b/theme/templates/admin/base_site.html @@ -97,7 +97,7 @@ SPDX-License-Identifier: GPL-3.0-or-later

Mediatek 2017-2020 — Nous contactez — - Explorer l'API + Explorer l'API

{% endif %} diff --git a/theme/templates/redoc.html b/theme/templates/redoc.html deleted file mode 100644 index 80e3107..0000000 --- a/theme/templates/redoc.html +++ /dev/null @@ -1,12 +0,0 @@ -{% extends "base.html" %} -{% comment %} -SPDX-License-Identifier: GPL-3.0-or-later -{% endcomment %} -{% load i18n static %} - -{% block coltype %}nopadding{% endblock %} - -{% block content %} - - -{% endblock %} From 1bf96683154e133fbdf05604bd93d8d9cdd18ed9 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 11 Feb 2020 21:28:57 +0100 Subject: [PATCH 50/79] Ignore venv and tox when compiling translations --- entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index aa2d908..2b7471d 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,7 @@ # This will launch the Django project as a fastcgi socket # then Apache or NGINX will be able to use that socket -python manage.py compilemessages +python manage.py compilemessages -i ".tox" -i "venv" python manage.py makemigrations # Wait for database From 98b38cd7a4a237fd53244dc55b6597b7ae53b697 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 11 Feb 2020 21:39:08 +0100 Subject: [PATCH 51/79] Fix entrypoint for Django 2.2 --- entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/entrypoint.sh b/entrypoint.sh index 2b7471d..bab3ff2 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -2,7 +2,9 @@ # This will launch the Django project as a fastcgi socket # then Apache or NGINX will be able to use that socket -python manage.py compilemessages -i ".tox" -i "venv" +# Option "-i" will be only available in Django 3.0+, but it does not support Python 3.5 +#python manage.py compilemessages -i ".tox" -i "venv" +python manage.py compilemessages python manage.py makemigrations # Wait for database From 4b07ddda23dac102b89316d70d6920d76122a0d7 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Wed, 12 Feb 2020 07:57:49 +0100 Subject: [PATCH 52/79] Django security upgrade --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2e68d39..22d851f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Django==2.2.4 +Django==2.2.10 docutils==0.14 Pillow==5.4.1 pytz==2019.1 From 39c3a598387b4281ceaaf2ae2a2fde2067705e49 Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Tue, 18 Feb 2020 21:34:12 +0100 Subject: [PATCH 53/79] Allow linter to fail --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 406cf3e..d1acd36 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,3 +24,4 @@ python37: linters: stage: test script: tox -e linters + allow_failure: true From 0c9b3c4d5f6df4495b79e4265b06d9231d9397e1 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Thu, 20 Feb 2020 13:42:22 +0100 Subject: [PATCH 54/79] =?UTF-8?q?Le=20Fetch=20ouvre=20ensuite=20la=20page?= =?UTF-8?q?=20pour=20entrer=20un=20nouveau=20m=C3=A9dia=20(code=20sans=20d?= =?UTF-8?q?oute=20temporaire)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media/forms.py | 7 ++++--- media/templates/media/isbn_button.html | 3 ++- tool_barcode_getblue.py | 3 +++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/media/forms.py b/media/forms.py index b3c3775..f904bf4 100644 --- a/media/forms.py +++ b/media/forms.py @@ -26,6 +26,8 @@ class MediaAdminForm(ModelForm): Download data from bedeteque :return True if success """ + print(self.request.POST) + print('isbn' in self.request.POST) scraper = BedetequeScraper() r = scraper.search_by_isbn(isbn) if not r: @@ -33,7 +35,6 @@ class MediaAdminForm(ModelForm): # If results, then take the most accurate data = scraper.scrap_bd_info(r[0]) self.cleaned_data.update(data) - print(self.cleaned_data) return True def download_data_openlibrary(self, isbn): @@ -93,7 +94,7 @@ class MediaAdminForm(ModelForm): """ super().clean() - if "_continue" in self.request.POST: + if "_isbn" in self.request.POST: isbn = self.cleaned_data.get('isbn') if isbn: # ISBN is present, try with bedeteque @@ -175,7 +176,7 @@ class MediaAdminForm(ModelForm): from django.core.exceptions import ValidationError try: # We don't want to check a field when we enter an ISBN. - if "_continue" not in self.request.POST \ + if "isbn" not in self.request.POST \ or not self.cleaned_data.get('isbn'): value = field.clean(value) self.cleaned_data[name] = value diff --git a/media/templates/media/isbn_button.html b/media/templates/media/isbn_button.html index 36bd9b2..1d1903d 100644 --- a/media/templates/media/isbn_button.html +++ b/media/templates/media/isbn_button.html @@ -1,3 +1,4 @@ {% load i18n %} {% include "django/forms/widgets/input.html" %} - \ No newline at end of file + + \ No newline at end of file diff --git a/tool_barcode_getblue.py b/tool_barcode_getblue.py index fd5b8cf..533414a 100644 --- a/tool_barcode_getblue.py +++ b/tool_barcode_getblue.py @@ -22,6 +22,9 @@ class Server(BaseHTTPRequestHandler): def do_GET(self): self._set_headers() isbn = self.path[7:-24] + if not isbn.isnumeric(): + print("Mauvais ISBN.") + return print("Hey j'ai un ISBN :", isbn) os.system("xdotool type " + isbn) os.system("xdotool key KP_Enter") From 4c55bdd2005e5a2443546f3acb3aa6f685ed9390 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Thu, 20 Feb 2020 13:52:50 +0100 Subject: [PATCH 55/79] =?UTF-8?q?Le=20Fetch=20ouvre=20ensuite=20la=20page?= =?UTF-8?q?=20pour=20entrer=20un=20nouveau=20m=C3=A9dia=20(code=20moins=20?= =?UTF-8?q?sale)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- media/forms.py | 7 ++- media/locale/fr/LC_MESSAGES/django.po | 84 ++++++++++++++------------ media/templates/media/isbn_button.html | 4 +- users/locale/fr/LC_MESSAGES/django.po | 68 ++++++++++----------- 4 files changed, 83 insertions(+), 80 deletions(-) diff --git a/media/forms.py b/media/forms.py index f904bf4..341cdf5 100644 --- a/media/forms.py +++ b/media/forms.py @@ -26,8 +26,6 @@ class MediaAdminForm(ModelForm): Download data from bedeteque :return True if success """ - print(self.request.POST) - print('isbn' in self.request.POST) scraper = BedetequeScraper() r = scraper.search_by_isbn(isbn) if not r: @@ -94,8 +92,11 @@ class MediaAdminForm(ModelForm): """ super().clean() - if "_isbn" in self.request.POST: + if "_isbn" in self.request.POST or "_isbn_addanother" in self.request.POST: isbn = self.cleaned_data.get('isbn') + if "_isbn_addanother" in self.request.POST: + self.request.POST = self.request.POST.copy() + self.request.POST['_addanother'] = 42 if isbn: # ISBN is present, try with bedeteque scrap_result = self.download_data_bedeteque(isbn) diff --git a/media/locale/fr/LC_MESSAGES/django.po b/media/locale/fr/LC_MESSAGES/django.po index 5bbae5c..d8e367a 100644 --- a/media/locale/fr/LC_MESSAGES/django.po +++ b/media/locale/fr/LC_MESSAGES/django.po @@ -3,7 +3,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2019-08-16 14:00+0200\n" +"POT-Creation-Date: 2020-02-20 13:50+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -13,7 +13,7 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" -#: admin.py:32 models.py:24 models.py:56 +#: admin.py:32 models.py:29 models.py:62 msgid "authors" msgstr "auteurs" @@ -25,7 +25,7 @@ msgstr "URL externe" msgid "Turn back" msgstr "Rendre" -#: admin.py:85 models.py:112 +#: admin.py:85 models.py:118 msgid "given back to" msgstr "rendu à" @@ -33,18 +33,18 @@ msgstr "rendu à" msgid "ISBN-10 or ISBN-13" msgstr "ISBN-10 ou ISBN-13" -#: models.py:16 models.py:136 +#: models.py:16 models.py:142 msgid "name" msgstr "nom" -#: models.py:23 -msgid "author" -msgstr "auteur" - -#: models.py:23 +#: models.py:21 msgid "note" msgstr "note" +#: models.py:28 +msgid "author" +msgstr "auteur" + #: models.py:35 msgid "ISBN" msgstr "ISBN" @@ -53,118 +53,124 @@ msgstr "ISBN" msgid "You may be able to scan it from a bar code." msgstr "Peut souvent être scanné à partir du code barre." -#: models.py:41 +#: models.py:42 msgid "title" msgstr "titre" -#: models.py:45 +#: models.py:46 msgid "subtitle" msgstr "sous-titre" -#: models.py:51 +#: models.py:52 msgid "external URL" msgstr "URL externe" -#: models.py:56 +#: models.py:57 msgid "side identifier" msgstr "côte" -#: models.py:64 +#: models.py:65 msgid "number of pages" msgstr "nombre de pages" -#: models.py:69 +#: models.py:70 msgid "publish date" msgstr "date de publication" -#: models.py:81 +#: models.py:82 msgid "medium" msgstr "medium" -#: models.py:82 +#: models.py:83 msgid "media" msgstr "media" -#: models.py:94 +#: models.py:95 msgid "borrower" msgstr "emprunteur" -#: models.py:97 +#: models.py:98 msgid "borrowed on" msgstr "emprunté le" -#: models.py:102 +#: models.py:103 msgid "given back on" msgstr "rendu le" -#: models.py:108 +#: models.py:109 msgid "borrowed with" msgstr "emprunté avec" -#: models.py:109 +#: models.py:110 msgid "The keyholder that registered this borrowed item." msgstr "Le permanencier qui enregistre cet emprunt." -#: models.py:118 +#: models.py:119 msgid "The keyholder to whom this item was given back." msgstr "Le permanencier à qui l'emprunt a été rendu." -#: models.py:125 +#: models.py:126 msgid "borrowed item" msgstr "emprunt" -#: models.py:126 +#: models.py:127 msgid "borrowed items" msgstr "emprunts" -#: models.py:146 +#: models.py:147 msgid "owner" msgstr "propriétaire" -#: models.py:151 +#: models.py:152 msgid "duration" msgstr "durée" -#: models.py:155 +#: models.py:156 msgid "minimum number of players" msgstr "nombre minimum de joueurs" -#: models.py:159 +#: models.py:160 msgid "maximum number of players" msgstr "nombre maximum de joueurs" -#: models.py:165 +#: models.py:166 msgid "comment" msgstr "commentaire" -#: models.py:172 +#: models.py:173 msgid "game" msgstr "jeu" -#: models.py:173 +#: models.py:174 msgid "games" msgstr "jeux" #: templates/media/isbn_button.html:3 -msgid "Fetch data" -msgstr "Télécharger les données" +msgid "Fetch data and add another" +msgstr "Télécharger les données et ajouter un nouveau medium" -#: validators.py:20 +#: templates/media/isbn_button.html:4 +#, fuzzy +#| msgid "Fetch data" +msgid "Fetch only" +msgstr "Télécharger uniquement les données" + +#: validators.py:19 msgid "Invalid ISBN: Not a string" msgstr "ISBN invalide : ce n'est pas une chaîne de caractères" -#: validators.py:23 +#: validators.py:22 msgid "Invalid ISBN: Wrong length" msgstr "ISBN invalide : mauvaise longueur" -#: validators.py:26 +#: validators.py:25 msgid "Invalid ISBN: Failed checksum" msgstr "ISBN invalide : mauvais checksum" -#: validators.py:29 +#: validators.py:28 msgid "Invalid ISBN: Only upper case allowed" msgstr "ISBN invalide : seulement les majuscules sont autorisées" -#: views.py:41 +#: views.py:44 msgid "Welcome to the Mediatek database" msgstr "Bienvenue sur la base de données de la Mediatek" diff --git a/media/templates/media/isbn_button.html b/media/templates/media/isbn_button.html index 1d1903d..91eb7e9 100644 --- a/media/templates/media/isbn_button.html +++ b/media/templates/media/isbn_button.html @@ -1,4 +1,4 @@ {% load i18n %} {% include "django/forms/widgets/input.html" %} - - \ No newline at end of file + + \ No newline at end of file diff --git a/users/locale/fr/LC_MESSAGES/django.po b/users/locale/fr/LC_MESSAGES/django.po index aa6e761..860ab62 100644 --- a/users/locale/fr/LC_MESSAGES/django.po +++ b/users/locale/fr/LC_MESSAGES/django.po @@ -3,7 +3,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2019-08-10 16:20+0200\n" +"POT-Creation-Date: 2020-02-20 13:51+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -13,39 +13,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" -#: admin.py:32 +#: admin.py:25 msgid "membership status" msgstr "statut adhérent" -#: admin.py:37 +#: admin.py:30 msgid "Yes" msgstr "Oui" -#: admin.py:54 +#: admin.py:47 msgid "Personal info" -msgstr "" +msgstr "Informations personnelles" -#: admin.py:56 +#: admin.py:49 msgid "Permissions" msgstr "" -#: admin.py:59 +#: admin.py:52 msgid "Important dates" -msgstr "" +msgstr "Dates importantes" -#: admin.py:89 +#: admin.py:82 msgid "An email to set the password was sent." msgstr "Un mail pour initialiser le mot de passe a été envoyé." -#: admin.py:92 +#: admin.py:85 msgid "The email is invalid." msgstr "L'adresse mail est invalide." -#: admin.py:111 +#: admin.py:103 msgid "Adhere" msgstr "Adhérer" -#: admin.py:114 +#: admin.py:106 msgid "is member" msgstr "statut adhérent" @@ -69,7 +69,7 @@ msgstr "emprunts maximal" msgid "Maximal amount of simultaneous borrowed item authorized." msgstr "Nombre maximal d'objets empruntés en même temps." -#: models.py:33 models.py:67 +#: models.py:33 msgid "comment" msgstr "commentaire" @@ -82,46 +82,30 @@ msgid "date joined" msgstr "" #: models.py:55 -msgid "name" -msgstr "nom" - -#: models.py:62 -msgid "owner" -msgstr "propriétaire" - -#: models.py:74 -msgid "key" -msgstr "clé" - -#: models.py:75 -msgid "keys" -msgstr "clés" - -#: models.py:80 msgid "starting in" msgstr "commence en" -#: models.py:81 +#: models.py:56 msgid "Year in which the membership year starts." msgstr "Année dans laquelle la plage d'adhésion commence." -#: models.py:85 +#: models.py:60 msgid "ending in" msgstr "finie en" -#: models.py:86 +#: models.py:61 msgid "Year in which the membership year ends." msgstr "Année dans laquelle la plage d'adhésion finie." -#: models.py:91 +#: models.py:66 msgid "members" msgstr "adhérents" -#: models.py:96 +#: models.py:71 msgid "membership year" msgstr "année d'adhésion" -#: models.py:97 +#: models.py:72 msgid "membership years" msgstr "années d'adhésion" @@ -133,6 +117,18 @@ msgstr "" msgid "Save" msgstr "" -#: views.py:40 +#: views.py:43 msgid "Edit user profile" msgstr "Editer le profil utilisateur" + +#~ msgid "name" +#~ msgstr "nom" + +#~ msgid "owner" +#~ msgstr "propriétaire" + +#~ msgid "key" +#~ msgstr "clé" + +#~ msgid "keys" +#~ msgstr "clés" From 10417242f4fbcfb6bf9766bcd4a7d35453daf2be Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Thu, 20 Feb 2020 17:32:50 +0100 Subject: [PATCH 56/79] Suspension temporaire des checksums invalides --- media/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/media/validators.py b/media/validators.py index d528ed1..151c90f 100644 --- a/media/validators.py +++ b/media/validators.py @@ -21,8 +21,8 @@ def isbn_validator(raw_isbn): if len(isbn_to_check) != 10 and len(isbn_to_check) != 13: raise ValidationError(_('Invalid ISBN: Wrong length')) - if not isbn.is_valid(isbn_to_check): - raise ValidationError(_('Invalid ISBN: Failed checksum')) + #if not isbn.is_valid(isbn_to_check): + # raise ValidationError(_('Invalid ISBN: Failed checksum')) if isbn_to_check != isbn_to_check.upper(): raise ValidationError(_('Invalid ISBN: Only upper case allowed')) From 8e39f6039e12cef150d9f31221e859c9aa487065 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sat, 22 Feb 2020 18:11:50 +0100 Subject: [PATCH 57/79] Prepare CAS authentication --- med/settings.py | 7 +++++++ med/settings_local.example.py | 4 ++++ med/urls.py | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/med/settings.py b/med/settings.py index b9b08cd..fb11d68 100644 --- a/med/settings.py +++ b/med/settings.py @@ -35,6 +35,7 @@ INSTALLED_APPS = [ # External apps 'reversion', 'rest_framework', + 'cas', # Django contrib 'django.contrib.admin', @@ -64,8 +65,14 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.contrib.sites.middleware.CurrentSiteMiddleware', + 'cas.middleware.CASMiddleware', ] +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'cas.backends.CASBackend', +) + ROOT_URLCONF = 'med.urls' TEMPLATES = [ diff --git a/med/settings_local.example.py b/med/settings_local.example.py index 51fb051..c2006a2 100644 --- a/med/settings_local.example.py +++ b/med/settings_local.example.py @@ -16,6 +16,10 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' SERVER_EMAIL = 'club-med@crans.org' +CAS_SERVER_URL = "https://note.crans.org/cas/" +CAS_LOGOUT_COMPLETELY = True +CAS_PROVIDE_URL_TO_LOGOUT = True + # Security settings SECURE_CONTENT_TYPE_NOSNIFF = False SECURE_BROWSER_XSS_FILTER = False diff --git a/med/urls.py b/med/urls.py index 51e3f3d..ad4bff8 100644 --- a/med/urls.py +++ b/med/urls.py @@ -2,6 +2,7 @@ # Copyright (C) 2017-2019 by BDE ENS Paris-Saclay # SPDX-License-Identifier: GPL-3.0-or-later +from cas import views as cas_views from django.contrib.auth.decorators import login_required from django.contrib.auth.views import PasswordResetView from django.urls import include, path @@ -41,4 +42,9 @@ urlpatterns = [ path('accounts/profile/', RedirectView.as_view(pattern_name='index')), path('database/doc/', include('django.contrib.admindocs.urls')), path('database/', admin_site.urls), + + # Include CAS authentication + # TODO Uncomment when NK20 will be ready + # path('admin/login/', cas_views.login, name='login'), + # path('admin/logout/', cas_views.logout, name='logout'), ] From aa9b69f2d671358acc78f2cc9c5ba9b578745974 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 14:56:31 +0200 Subject: [PATCH 58/79] Fix openlibrary scrap, fix linters --- med/settings.py | 2 +- media/forms.py | 29 +++++++++++++++++++++++++---- media/validators.py | 3 +-- requirements.txt | 4 ++-- tox.ini | 6 +++--- 5 files changed, 32 insertions(+), 12 deletions(-) diff --git a/med/settings.py b/med/settings.py index fb11d68..7e79e27 100644 --- a/med/settings.py +++ b/med/settings.py @@ -65,7 +65,7 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.contrib.sites.middleware.CurrentSiteMiddleware', - 'cas.middleware.CASMiddleware', + # 'cas.middleware.CASMiddleware', ] AUTHENTICATION_BACKENDS = ( diff --git a/media/forms.py b/media/forms.py index 341cdf5..fd02718 100644 --- a/media/forms.py +++ b/media/forms.py @@ -54,6 +54,7 @@ class MediaAdminForm(ModelForm): return False def parse_data_openlibrary(self, data): + print(data) self.cleaned_data['external_url'] = data['url'] if 'title' in data: self.cleaned_data['title'] = data['title'] @@ -74,9 +75,28 @@ class MediaAdminForm(ModelForm): if len(split) == 1: self.cleaned_data['publish_date'] = split[0] + "-01-01" else: - self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ - .format(split[2], months.index(split[0]) + 1, - int(split[1]), ) + month_to_number = dict( + Jan="01", + Feb="02", + Mar="03", + Apr="04", + May="05", + Jun="06", + Jul="07", + Aug="08", + Sep="09", + Oct="10", + Nov="11", + Dec="12", + ) + if split[0] in month_to_number: + self.cleaned_data['publish_date']\ + = split[2] + "-" \ + + month_to_number[split[0]] + "-" + split[1] + else: + self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ + .format(split[2], months.index(split[0]) + + 1, int(split[1]), ) if 'authors' in data: if 'author' not in self.cleaned_data: @@ -92,7 +112,8 @@ class MediaAdminForm(ModelForm): """ super().clean() - if "_isbn" in self.request.POST or "_isbn_addanother" in self.request.POST: + if "_isbn" in self.request.POST\ + or "_isbn_addanother" in self.request.POST: isbn = self.cleaned_data.get('isbn') if "_isbn_addanother" in self.request.POST: self.request.POST = self.request.POST.copy() diff --git a/media/validators.py b/media/validators.py index 151c90f..3d7af51 100644 --- a/media/validators.py +++ b/media/validators.py @@ -8,7 +8,6 @@ Based on https://github.com/secnot/django-isbn-field from django.core.exceptions import ValidationError from django.utils.translation import ugettext_lazy as _ -from stdnum import isbn def isbn_validator(raw_isbn): @@ -21,7 +20,7 @@ def isbn_validator(raw_isbn): if len(isbn_to_check) != 10 and len(isbn_to_check) != 13: raise ValidationError(_('Invalid ISBN: Wrong length')) - #if not isbn.is_valid(isbn_to_check): + # if not isbn.is_valid(isbn_to_check): # raise ValidationError(_('Invalid ISBN: Failed checksum')) if isbn_to_check != isbn_to_check.upper(): diff --git a/requirements.txt b/requirements.txt index 22d851f..73c63c8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ Pillow==5.4.1 pytz==2019.1 six==1.12.0 sqlparse==0.2.4 +django-cas-client==1.5.3 django-reversion==3.0.3 python-stdnum==1.10 djangorestframework==3.9.2 pyyaml==3.13 coreapi==2.3.3 -psycopg2==2.7.7 +psycopg2-binary uwsgi==2.0.18 -mysqlclient==1.4.3 diff --git a/tox.ini b/tox.ini index a512056..ebc489d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py35,py36,py37,linters +envlist = py35,py36,py37,py38,linters skipsdist = True [testenv] @@ -28,11 +28,11 @@ deps = pyflakes pylint commands = - flake8 logs media search users + flake8 logs media users pylint . [flake8] -ignore = D203, W503, E203, I100, I201, I202 +ignore = D203, W503, E203, I100, I201, I202, C901 exclude = .tox, .git, From 43b3b5ccfe6343ba77598a3aa7e694050bd07234 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 15:09:06 +0200 Subject: [PATCH 59/79] Fix CI --- .gitlab-ci.yml | 7 ++++++- media/forms.py | 2 +- media/tests/test_templates.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d1acd36..449eecf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: python:3.6 +image: python:3.8 stages: - test @@ -21,6 +21,11 @@ python37: stage: test script: tox -e py37 +python38: + image: python:3.8 + stage: test + script: tox -e py37 + linters: stage: test script: tox -e linters diff --git a/media/forms.py b/media/forms.py index fd02718..8f767cf 100644 --- a/media/forms.py +++ b/media/forms.py @@ -198,7 +198,7 @@ class MediaAdminForm(ModelForm): from django.core.exceptions import ValidationError try: # We don't want to check a field when we enter an ISBN. - if "isbn" not in self.request.POST \ + if "_isbn" not in self.request.POST \ or not self.cleaned_data.get('isbn'): value = field.clean(value) self.cleaned_data[name] = value diff --git a/media/tests/test_templates.py b/media/tests/test_templates.py index b9d2e75..0a1a150 100644 --- a/media/tests/test_templates.py +++ b/media/tests/test_templates.py @@ -47,7 +47,7 @@ class TemplateTests(TestCase): def test_media_isbn_download(self): data = { - '_continue': True, + '_isbn': True, 'isbn': "0316358525", } response = self.client.post(reverse( From adbaf664010e484cb1eb0fcd7e4460319e89d3c6 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 17:08:40 +0200 Subject: [PATCH 60/79] Scrap from Google books API --- media/forms.py | 69 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 8 deletions(-) diff --git a/media/forms.py b/media/forms.py index 8f767cf..1933ae8 100644 --- a/media/forms.py +++ b/media/forms.py @@ -8,6 +8,7 @@ import unicodedata import urllib.request from django.forms import ModelForm +from django.utils.translation import gettext_lazy as _ from .models import Auteur from .scraper import BedetequeScraper @@ -35,6 +36,51 @@ class MediaAdminForm(ModelForm): self.cleaned_data.update(data) return True + def download_data_google(self, isbn): + """ + Download data from google books + :return True if success + """ + api_url = "https://www.googleapis.com/books/v1/volumes?q=isbn:{}".format(isbn) + with urllib.request.urlopen(api_url) as url: + data = json.loads(url.read().decode()) + + if data and data['totalItems']: + data = data['items'][0] + # Fill the data + self.parse_data_google(data) + return True + return False + + def parse_data_google(self, data): + print(data) + info = data['volumeInfo'] + self.cleaned_data['external_url'] = info['canonicalVolumeLink'] + if 'title' in info: + self.cleaned_data['title'] = info['title'] + if 'subtitle' in data: + self.cleaned_data['subtitle'] = info['subtitle'] + + if 'pageCount' in info: + self.cleaned_data['number_of_pages'] = \ + info['pageCount'] + elif not self.cleaned_data['number_of_pages']: + self.cleaned_data['number_of_pages'] = 0 + + if 'publishedDate' in info: + self.cleaned_data['publish_date'] = info['publishedDate'] + + if 'authors' not in self.cleaned_data: + self.cleaned_data['authors'] = list() + + if 'authors' in info: + for author in info['authors']: + author_obj = Auteur.objects.get_or_create( + name=author)[0] + self.cleaned_data['authors'].append(author_obj) + + print(self.cleaned_data) + def download_data_openlibrary(self, isbn): """ Download data from openlibrary @@ -54,7 +100,6 @@ class MediaAdminForm(ModelForm): return False def parse_data_openlibrary(self, data): - print(data) self.cleaned_data['external_url'] = data['url'] if 'title' in data: self.cleaned_data['title'] = data['title'] @@ -89,18 +134,19 @@ class MediaAdminForm(ModelForm): Nov="11", Dec="12", ) - if split[0] in month_to_number: + if split[0][:3] in month_to_number: self.cleaned_data['publish_date']\ = split[2] + "-" \ - + month_to_number[split[0]] + "-" + split[1] + + month_to_number[split[0][:3]] + "-" + split[1] else: self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \ .format(split[2], months.index(split[0]) + 1, int(split[1]), ) + if 'authors' not in self.cleaned_data: + self.cleaned_data['authors'] = list() + if 'authors' in data: - if 'author' not in self.cleaned_data: - self.cleaned_data['authors'] = list() for author in data['authors']: author_obj = Auteur.objects.get_or_create( name=author['name'])[0] @@ -122,8 +168,13 @@ class MediaAdminForm(ModelForm): # ISBN is present, try with bedeteque scrap_result = self.download_data_bedeteque(isbn) if not scrap_result: - # Try with OpenLibrary - self.download_data_openlibrary(isbn) + # Try with Fnac + scrap_result = self.download_data_google(isbn) + if not scrap_result: + # Try with OpenLibrary + if not self.download_data_openlibrary(isbn): + self.add_error('isbn', _("This ISBN is not found.")) + return self.cleaned_data if self.cleaned_data['title']: self.cleaned_data['title'] = re.sub( @@ -183,6 +234,8 @@ class MediaAdminForm(ModelForm): ).casefold().upper() self.cleaned_data['side_identifier'] = side_identifier + print(self.cleaned_data) + return self.cleaned_data def _clean_fields(self): @@ -198,7 +251,7 @@ class MediaAdminForm(ModelForm): from django.core.exceptions import ValidationError try: # We don't want to check a field when we enter an ISBN. - if "_isbn" not in self.request.POST \ + if "isbn" not in self.request.POST \ or not self.cleaned_data.get('isbn'): value = field.clean(value) self.cleaned_data[name] = value From ea30cdec6efa5761696c6a2a474fafab21d01a46 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 17:33:32 +0200 Subject: [PATCH 61/79] Add a model to store future media list. --- media/admin.py | 19 ++++++++++++++++++- media/forms.py | 4 +--- media/models.py | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/media/admin.py b/media/admin.py index a18fabd..6c6a07c 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, Emprunt, Jeu, Media +from .models import Auteur, Emprunt, Jeu, Media, FutureMedia class AuteurAdmin(VersionAdmin): @@ -60,6 +60,22 @@ class MediaAdmin(VersionAdmin): extra_context=extra_context) +class FutureMediaAdmin(VersionAdmin): + list_display = ('isbn',) + search_fields = ('isbn',) + + def changeform_view(self, request, object_id=None, form_url='', + extra_context=None): + """ + We use _continue for ISBN fetching, so remove continue button + """ + extra_context = extra_context or {} + extra_context['show_save_and_continue'] = False + extra_context['show_save'] = False + return super().changeform_view(request, object_id, form_url, + extra_context=extra_context) + + class EmpruntAdmin(VersionAdmin): list_display = ('media', 'user', 'date_emprunt', 'date_rendu', 'permanencier_emprunt', 'permanencier_rendu_custom') @@ -105,5 +121,6 @@ class JeuAdmin(VersionAdmin): admin_site.register(Auteur, AuteurAdmin) admin_site.register(Media, MediaAdmin) +admin_site.register(FutureMedia, FutureMediaAdmin) admin_site.register(Emprunt, EmpruntAdmin) admin_site.register(Jeu, JeuAdmin) diff --git a/media/forms.py b/media/forms.py index 1933ae8..224dd0c 100644 --- a/media/forms.py +++ b/media/forms.py @@ -41,7 +41,7 @@ class MediaAdminForm(ModelForm): Download data from google books :return True if success """ - api_url = "https://www.googleapis.com/books/v1/volumes?q=isbn:{}".format(isbn) + api_url = "https://www.googleapis.com/books/v1/volumes?q=ISBN:{}".format(isbn) with urllib.request.urlopen(api_url) as url: data = json.loads(url.read().decode()) @@ -234,8 +234,6 @@ class MediaAdminForm(ModelForm): ).casefold().upper() self.cleaned_data['side_identifier'] = side_identifier - print(self.cleaned_data) - return self.cleaned_data def _clean_fields(self): diff --git a/media/models.py b/media/models.py index dbfaf04..3a67389 100644 --- a/media/models.py +++ b/media/models.py @@ -84,6 +84,23 @@ class Media(models.Model): ordering = ['title', 'subtitle'] +class FutureMedia(models.Model): + isbn = ISBNField( + _('ISBN'), + help_text=_('You may be able to scan it from a bar code.'), + unique=True, + blank=True, + null=True, + ) + + class Meta: + verbose_name = _("future medium") + verbose_name_plural = _("future media") + + def __str__(self): + return "Future medium (ISBN: {isbn})".format(isbn=self.isbn, ) + + class Emprunt(models.Model): media = models.ForeignKey( 'Media', From 1c8d5750bb877bcc383258fa968dad6f238d80fe Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 17:34:53 +0200 Subject: [PATCH 62/79] Fix linters --- med/admin.py | 1 - media/admin.py | 2 +- media/forms.py | 6 ++++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/med/admin.py b/med/admin.py index e6365ff..acb795d 100644 --- a/med/admin.py +++ b/med/admin.py @@ -7,7 +7,6 @@ from django.contrib.auth.admin import Group, GroupAdmin from django.contrib.sites.admin import Site, SiteAdmin from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache - from media.models import Emprunt diff --git a/media/admin.py b/media/admin.py index 6c6a07c..23432ec 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, Emprunt, Jeu, Media, FutureMedia +from .models import Auteur, Emprunt, FutureMedia, Jeu, Media class AuteurAdmin(VersionAdmin): diff --git a/media/forms.py b/media/forms.py index 224dd0c..2a46052 100644 --- a/media/forms.py +++ b/media/forms.py @@ -41,7 +41,8 @@ class MediaAdminForm(ModelForm): Download data from google books :return True if success """ - api_url = "https://www.googleapis.com/books/v1/volumes?q=ISBN:{}".format(isbn) + api_url = "https://www.googleapis.com/books/v1/volumes?q=ISBN:{}"\ + .format(isbn) with urllib.request.urlopen(api_url) as url: data = json.loads(url.read().decode()) @@ -173,7 +174,8 @@ class MediaAdminForm(ModelForm): if not scrap_result: # Try with OpenLibrary if not self.download_data_openlibrary(isbn): - self.add_error('isbn', _("This ISBN is not found.")) + self.add_error('isbn', + _("This ISBN is not found.")) return self.cleaned_data if self.cleaned_data['title']: From 1657f5c42c5603bcdfa1e6921ee8efbfc6c32323 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Tue, 12 May 2020 17:43:14 +0200 Subject: [PATCH 63/79] Update translations --- entrypoint.sh | 2 +- media/locale/fr/LC_MESSAGES/django.po | 64 +++++++++++++++------------ 2 files changed, 37 insertions(+), 29 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index bab3ff2..ed17c5f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -28,4 +28,4 @@ uwsgi --chdir="$(pwd)" \ --max-requests=5000 \ --vacuum \ --daemonize="$(pwd)/uwsgi.log" \ - --protocol=fastcgi \ No newline at end of file + --protocol=fastcgi diff --git a/media/locale/fr/LC_MESSAGES/django.po b/media/locale/fr/LC_MESSAGES/django.po index d8e367a..c75a7d3 100644 --- a/media/locale/fr/LC_MESSAGES/django.po +++ b/media/locale/fr/LC_MESSAGES/django.po @@ -3,7 +3,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2020-02-20 13:50+0100\n" +"POT-Creation-Date: 2020-05-12 17:42+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -21,11 +21,11 @@ msgstr "auteurs" msgid "external url" msgstr "URL externe" -#: admin.py:82 +#: admin.py:98 msgid "Turn back" msgstr "Rendre" -#: admin.py:85 models.py:118 +#: admin.py:101 models.py:135 msgid "given back to" msgstr "rendu à" @@ -33,7 +33,11 @@ msgstr "rendu à" msgid "ISBN-10 or ISBN-13" msgstr "ISBN-10 ou ISBN-13" -#: models.py:16 models.py:142 +#: forms.py:178 +msgid "This ISBN is not found." +msgstr "L'ISBN n'a pas été trouvé." + +#: models.py:16 models.py:159 msgid "name" msgstr "nom" @@ -45,11 +49,11 @@ msgstr "note" msgid "author" msgstr "auteur" -#: models.py:35 +#: models.py:35 models.py:89 msgid "ISBN" msgstr "ISBN" -#: models.py:36 +#: models.py:36 models.py:90 msgid "You may be able to scan it from a bar code." msgstr "Peut souvent être scanné à partir du code barre." @@ -85,63 +89,71 @@ msgstr "medium" msgid "media" msgstr "media" -#: models.py:95 +#: models.py:97 +msgid "future medium" +msgstr "medium à importer" + +#: models.py:98 +msgid "future media" +msgstr "medias à importer" + +#: models.py:112 msgid "borrower" msgstr "emprunteur" -#: models.py:98 +#: models.py:115 msgid "borrowed on" msgstr "emprunté le" -#: models.py:103 +#: models.py:120 msgid "given back on" msgstr "rendu le" -#: models.py:109 +#: models.py:126 msgid "borrowed with" msgstr "emprunté avec" -#: models.py:110 +#: models.py:127 msgid "The keyholder that registered this borrowed item." msgstr "Le permanencier qui enregistre cet emprunt." -#: models.py:119 +#: models.py:136 msgid "The keyholder to whom this item was given back." msgstr "Le permanencier à qui l'emprunt a été rendu." -#: models.py:126 +#: models.py:143 msgid "borrowed item" msgstr "emprunt" -#: models.py:127 +#: models.py:144 msgid "borrowed items" msgstr "emprunts" -#: models.py:147 +#: models.py:164 msgid "owner" msgstr "propriétaire" -#: models.py:152 +#: models.py:169 msgid "duration" msgstr "durée" -#: models.py:156 +#: models.py:173 msgid "minimum number of players" msgstr "nombre minimum de joueurs" -#: models.py:160 +#: models.py:177 msgid "maximum number of players" msgstr "nombre maximum de joueurs" -#: models.py:166 +#: models.py:183 msgid "comment" msgstr "commentaire" -#: models.py:173 +#: models.py:190 msgid "game" msgstr "jeu" -#: models.py:174 +#: models.py:191 msgid "games" msgstr "jeux" @@ -155,19 +167,15 @@ msgstr "Télécharger les données et ajouter un nouveau medium" msgid "Fetch only" msgstr "Télécharger uniquement les données" -#: validators.py:19 +#: validators.py:18 msgid "Invalid ISBN: Not a string" msgstr "ISBN invalide : ce n'est pas une chaîne de caractères" -#: validators.py:22 +#: validators.py:21 msgid "Invalid ISBN: Wrong length" msgstr "ISBN invalide : mauvaise longueur" -#: validators.py:25 -msgid "Invalid ISBN: Failed checksum" -msgstr "ISBN invalide : mauvais checksum" - -#: validators.py:28 +#: validators.py:27 msgid "Invalid ISBN: Only upper case allowed" msgstr "ISBN invalide : seulement les majuscules sont autorisées" From 82efeba272bbc20b173d7e68007752f153fbfee1 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Thu, 21 May 2020 16:56:41 +0200 Subject: [PATCH 64/79] Split comic strips and mangas --- entrypoint.sh | 2 + media/admin.py | 3 +- media/forms.py | 25 ++++--- media/management/__init__.py | 0 media/management/commands/__init__.py | 0 .../management/commands/split_media_types.py | 50 ++++++++++++++ media/migrations/0027_futuremedia.py | 26 +++++++ media/migrations/0028_manga.py | 34 ++++++++++ media/models.py | 68 +++++++++++++++++++ media/scraper.py | 6 ++ 10 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 media/management/__init__.py create mode 100644 media/management/commands/__init__.py create mode 100644 media/management/commands/split_media_types.py create mode 100644 media/migrations/0027_futuremedia.py create mode 100644 media/migrations/0028_manga.py diff --git a/entrypoint.sh b/entrypoint.sh index ed17c5f..50b05fe 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,6 +13,8 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input +python manage.py runserver 0.0.0.0:8000 + # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped diff --git a/media/admin.py b/media/admin.py index 23432ec..e679f91 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, Emprunt, FutureMedia, Jeu, Media +from .models import Auteur, Emprunt, FutureMedia, Jeu, Media, Manga class AuteurAdmin(VersionAdmin): @@ -121,6 +121,7 @@ class JeuAdmin(VersionAdmin): admin_site.register(Auteur, AuteurAdmin) admin_site.register(Media, MediaAdmin) +admin_site.register(Manga, MediaAdmin) admin_site.register(FutureMedia, FutureMediaAdmin) admin_site.register(Emprunt, EmpruntAdmin) admin_site.register(Jeu, JeuAdmin) diff --git a/media/forms.py b/media/forms.py index 2a46052..0d9b0e4 100644 --- a/media/forms.py +++ b/media/forms.py @@ -10,7 +10,7 @@ import urllib.request from django.forms import ModelForm from django.utils.translation import gettext_lazy as _ -from .models import Auteur +from .models import Auteur, Media from .scraper import BedetequeScraper @@ -54,7 +54,6 @@ class MediaAdminForm(ModelForm): return False def parse_data_google(self, data): - print(data) info = data['volumeInfo'] self.cleaned_data['external_url'] = info['canonicalVolumeLink'] if 'title' in info: @@ -71,7 +70,7 @@ class MediaAdminForm(ModelForm): if 'publishedDate' in info: self.cleaned_data['publish_date'] = info['publishedDate'] - if 'authors' not in self.cleaned_data: + if 'authors' not in self.cleaned_data or not self.cleaned_data['authors']: self.cleaned_data['authors'] = list() if 'authors' in info: @@ -144,7 +143,7 @@ class MediaAdminForm(ModelForm): .format(split[2], months.index(split[0]) + 1, int(split[1]), ) - if 'authors' not in self.cleaned_data: + if 'authors' not in self.cleaned_data or not self.cleaned_data['authors']: self.cleaned_data['authors'] = list() if 'authors' in data: @@ -159,17 +158,17 @@ class MediaAdminForm(ModelForm): """ super().clean() - if "_isbn" in self.request.POST\ - or "_isbn_addanother" in self.request.POST: + if "_isbn" in self.data\ + or "_isbn_addanother" in self.data: isbn = self.cleaned_data.get('isbn') - if "_isbn_addanother" in self.request.POST: - self.request.POST = self.request.POST.copy() - self.request.POST['_addanother'] = 42 + if "_isbn_addanother" in self.data: + self.data = self.data.copy() + self.data['_addanother'] = 42 if isbn: # ISBN is present, try with bedeteque scrap_result = self.download_data_bedeteque(isbn) if not scrap_result: - # Try with Fnac + # Try with Google scrap_result = self.download_data_google(isbn) if not scrap_result: # Try with OpenLibrary @@ -251,7 +250,7 @@ class MediaAdminForm(ModelForm): from django.core.exceptions import ValidationError try: # We don't want to check a field when we enter an ISBN. - if "isbn" not in self.request.POST \ + if "isbn" not in self.data \ or not self.cleaned_data.get('isbn'): value = field.clean(value) self.cleaned_data[name] = value @@ -260,3 +259,7 @@ class MediaAdminForm(ModelForm): self.cleaned_data[name] = value except ValidationError as e: self.add_error(name, e) + + class Meta: + model = Media + fields = '__all__' diff --git a/media/management/__init__.py b/media/management/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/media/management/commands/__init__.py b/media/management/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/media/management/commands/split_media_types.py b/media/management/commands/split_media_types.py new file mode 100644 index 0000000..7e30a8a --- /dev/null +++ b/media/management/commands/split_media_types.py @@ -0,0 +1,50 @@ +from django.core.management import BaseCommand + +from media.forms import MediaAdminForm +from media.models import Media, Manga + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--view-only', action="store_true", + help="Display only modifications. " + + "Only useful for debug.") + + def handle(self, *args, **options): + converted = 0 + + for media in Media.objects.all(): + self.stdout.write(str(media)) + form = MediaAdminForm(instance=media, data={"isbn": media.isbn, "_isbn": True, }) + form.full_clean() + + if not "format" in form.cleaned_data: + self.stdout.write("Format not specified. Assume it is a comic strip.") + continue + + format = form.cleaned_data["format"] + self.stdout.write("Format: {}".format(format)) + + if not options["view_only"]: + if format == "manga": + self.stdout.write(self.style.WARNING("This media is a manga. Transfer it into a new object...")) + manga = Manga.objects.create( + isbn=media.isbn, + title=media.title, + subtitle=media.subtitle, + external_url=media.external_url, + side_identifier=media.side_identifier, + number_of_pages=media.number_of_pages, + publish_date=media.publish_date, + ) + + manga.authors.set(media.authors.all()) + manga.save() + + self.stdout.write(self.style.SUCCESS("Manga successfully saved. Deleting old medium...")) + + media.delete() + self.stdout.write(self.style.SUCCESS("Medium deleted")) + + converted += 1 + self.stdout.write(self.style.SUCCESS("Successfully saved {:d} mangas".format(converted))) diff --git a/media/migrations/0027_futuremedia.py b/media/migrations/0027_futuremedia.py new file mode 100644 index 0000000..13ef53c --- /dev/null +++ b/media/migrations/0027_futuremedia.py @@ -0,0 +1,26 @@ +# Generated by Django 2.2.10 on 2020-05-12 15:23 + +from django.db import migrations, models +import media.fields +import media.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0026_auto_20200210_1740'), + ] + + operations = [ + migrations.CreateModel( + name='FutureMedia', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('isbn', media.fields.ISBNField(blank=True, help_text='You may be able to scan it from a bar code.', max_length=28, null=True, unique=True, validators=[media.validators.isbn_validator], verbose_name='ISBN')), + ], + options={ + 'verbose_name': 'future medium', + 'verbose_name_plural': 'future media', + }, + ), + ] diff --git a/media/migrations/0028_manga.py b/media/migrations/0028_manga.py new file mode 100644 index 0000000..2afd409 --- /dev/null +++ b/media/migrations/0028_manga.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.10 on 2020-05-21 14:28 + +from django.db import migrations, models +import media.fields +import media.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0027_futuremedia'), + ] + + operations = [ + migrations.CreateModel( + name='Manga', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('isbn', media.fields.ISBNField(blank=True, help_text='You may be able to scan it from a bar code.', max_length=28, null=True, unique=True, validators=[media.validators.isbn_validator], verbose_name='ISBN')), + ('title', models.CharField(max_length=255, verbose_name='title')), + ('subtitle', models.CharField(blank=True, max_length=255, null=True, verbose_name='subtitle')), + ('external_url', models.URLField(blank=True, null=True, verbose_name='external URL')), + ('side_identifier', models.CharField(max_length=255, verbose_name='side identifier')), + ('number_of_pages', models.PositiveIntegerField(blank=True, null=True, verbose_name='number of pages')), + ('publish_date', models.DateField(blank=True, null=True, verbose_name='publish date')), + ('authors', models.ManyToManyField(to='media.Auteur', verbose_name='authors')), + ], + options={ + 'verbose_name': 'medium', + 'verbose_name_plural': 'media', + 'ordering': ['title', 'subtitle'], + }, + ), + ] diff --git a/media/models.py b/media/models.py index 3a67389..60f7c2f 100644 --- a/media/models.py +++ b/media/models.py @@ -38,34 +38,41 @@ class Media(models.Model): blank=True, null=True, ) + title = models.CharField( verbose_name=_('title'), max_length=255, ) + subtitle = models.CharField( verbose_name=_('subtitle'), max_length=255, blank=True, null=True, ) + external_url = models.URLField( verbose_name=_('external URL'), blank=True, null=True, ) + side_identifier = models.CharField( verbose_name=_('side identifier'), max_length=255, ) + authors = models.ManyToManyField( 'Auteur', verbose_name=_('authors'), ) + number_of_pages = models.PositiveIntegerField( verbose_name=_('number of pages'), blank=True, null=True, ) + publish_date = models.DateField( verbose_name=_('publish date'), blank=True, @@ -84,6 +91,67 @@ class Media(models.Model): ordering = ['title', 'subtitle'] +class Manga(models.Model): + isbn = ISBNField( + _('ISBN'), + help_text=_('You may be able to scan it from a bar code.'), + unique=True, + blank=True, + null=True, + ) + + title = models.CharField( + verbose_name=_('title'), + max_length=255, + ) + + subtitle = models.CharField( + verbose_name=_('subtitle'), + max_length=255, + blank=True, + null=True, + ) + + external_url = models.URLField( + verbose_name=_('external URL'), + blank=True, + null=True, + ) + + side_identifier = models.CharField( + verbose_name=_('side identifier'), + max_length=255, + ) + + authors = models.ManyToManyField( + 'Auteur', + verbose_name=_('authors'), + ) + + number_of_pages = models.PositiveIntegerField( + verbose_name=_('number of pages'), + blank=True, + null=True, + ) + + publish_date = models.DateField( + verbose_name=_('publish date'), + blank=True, + null=True, + ) + + def __str__(self): + if self.subtitle: + return "{} : {}".format(self.title, self.subtitle) + else: + return self.title + + class Meta: + verbose_name = _("manga") + verbose_name_plural = _("mangas") + ordering = ['title', 'subtitle'] + + class FutureMedia(models.Model): isbn = ISBNField( _('ISBN'), diff --git a/media/scraper.py b/media/scraper.py index 045a8b0..84de5a8 100644 --- a/media/scraper.py +++ b/media/scraper.py @@ -58,6 +58,7 @@ class BedetequeScraper: regex_subtitle = r'

\s*(.*)

' regex_publish_date = r'datePublished\" content=\"([\d-]*)\">' regex_nb_of_pages = r'numberOfPages\">(\d*)Format : Format (\w+)' regex_author = r'(((?!<).)*)' regex_illustrator = r'span itemprop=\"illustrator\">(((?!<).)*) Date: Thu, 21 May 2020 17:07:50 +0200 Subject: [PATCH 65/79] Fix synthax --- entrypoint.sh | 2 -- media/admin.py | 2 +- media/forms.py | 6 ++++-- .../management/commands/split_media_types.py | 20 ++++++++++++------- media/migrations/0029_auto_20200521_1659.py | 17 ++++++++++++++++ 5 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 media/migrations/0029_auto_20200521_1659.py diff --git a/entrypoint.sh b/entrypoint.sh index 50b05fe..ed17c5f 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,8 +13,6 @@ sleep 2 python manage.py migrate python manage.py collectstatic --no-input -python manage.py runserver 0.0.0.0:8000 - # harakiri parameter respawns processes taking more than 20 seconds # max-requests parameter respawns processes after serving 5000 requests # vacuum parameter cleans up when stopped diff --git a/media/admin.py b/media/admin.py index e679f91..f4e2d56 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, Emprunt, FutureMedia, Jeu, Media, Manga +from .models import Auteur, Emprunt, FutureMedia, Jeu, Manga, Media class AuteurAdmin(VersionAdmin): diff --git a/media/forms.py b/media/forms.py index 0d9b0e4..f368974 100644 --- a/media/forms.py +++ b/media/forms.py @@ -70,7 +70,8 @@ class MediaAdminForm(ModelForm): if 'publishedDate' in info: self.cleaned_data['publish_date'] = info['publishedDate'] - if 'authors' not in self.cleaned_data or not self.cleaned_data['authors']: + if 'authors' not in self.cleaned_data \ + or not self.cleaned_data['authors']: self.cleaned_data['authors'] = list() if 'authors' in info: @@ -143,7 +144,8 @@ class MediaAdminForm(ModelForm): .format(split[2], months.index(split[0]) + 1, int(split[1]), ) - if 'authors' not in self.cleaned_data or not self.cleaned_data['authors']: + if 'authors' not in self.cleaned_data \ + or not self.cleaned_data['authors']: self.cleaned_data['authors'] = list() if 'authors' in data: diff --git a/media/management/commands/split_media_types.py b/media/management/commands/split_media_types.py index 7e30a8a..b5d7d3c 100644 --- a/media/management/commands/split_media_types.py +++ b/media/management/commands/split_media_types.py @@ -1,7 +1,7 @@ from django.core.management import BaseCommand from media.forms import MediaAdminForm -from media.models import Media, Manga +from media.models import Manga, Media class Command(BaseCommand): @@ -15,11 +15,13 @@ class Command(BaseCommand): for media in Media.objects.all(): self.stdout.write(str(media)) - form = MediaAdminForm(instance=media, data={"isbn": media.isbn, "_isbn": True, }) + form = MediaAdminForm(instance=media, + data={"isbn": media.isbn, "_isbn": True, }) form.full_clean() - if not "format" in form.cleaned_data: - self.stdout.write("Format not specified. Assume it is a comic strip.") + if "format" not in form.cleaned_data: + self.stdout.write("Format not specified." + " Assume it is a comic strip.") continue format = form.cleaned_data["format"] @@ -27,7 +29,9 @@ class Command(BaseCommand): if not options["view_only"]: if format == "manga": - self.stdout.write(self.style.WARNING("This media is a manga. Transfer it into a new object...")) + self.stdout.write(self.style.WARNING( + "This media is a manga. " + "Transfer it into a new object...")) manga = Manga.objects.create( isbn=media.isbn, title=media.title, @@ -41,10 +45,12 @@ class Command(BaseCommand): manga.authors.set(media.authors.all()) manga.save() - self.stdout.write(self.style.SUCCESS("Manga successfully saved. Deleting old medium...")) + self.stdout.write(self.style.SUCCESS( + "Manga successfully saved. Deleting old medium...")) media.delete() self.stdout.write(self.style.SUCCESS("Medium deleted")) converted += 1 - self.stdout.write(self.style.SUCCESS("Successfully saved {:d} mangas".format(converted))) + self.stdout.write(self.style.SUCCESS( + "Successfully saved {:d} mangas".format(converted))) diff --git a/media/migrations/0029_auto_20200521_1659.py b/media/migrations/0029_auto_20200521_1659.py new file mode 100644 index 0000000..374eff4 --- /dev/null +++ b/media/migrations/0029_auto_20200521_1659.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.10 on 2020-05-21 14:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0028_manga'), + ] + + operations = [ + migrations.AlterModelOptions( + name='manga', + options={'ordering': ['title', 'subtitle'], 'verbose_name': 'manga', 'verbose_name_plural': 'mangas'}, + ), + ] From 054865cd41d909f59f9a702bf98e26b24449dba7 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 18:04:41 +0200 Subject: [PATCH 66/79] Ajout des CD et des vinyles --- media/admin.py | 17 +++++- media/forms.py | 4 +- .../management/commands/split_media_types.py | 10 ++- media/models.py | 61 +++++++++++++++---- media/serializers.py | 4 +- media/tests/test_templates.py | 6 +- media/views.py | 4 +- 7 files changed, 82 insertions(+), 24 deletions(-) diff --git a/media/admin.py b/media/admin.py index f4e2d56..bd92c6b 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, Emprunt, FutureMedia, Jeu, Manga, Media +from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga, Vinyle class AuteurAdmin(VersionAdmin): @@ -76,6 +76,17 @@ class FutureMediaAdmin(VersionAdmin): extra_context=extra_context) +class CDAdmin(VersionAdmin): + list_display = ('title', 'authors_list', 'side_identifier',) + search_fields = ('title', 'authors__name', 'side_identifier',) + autocomplete_fields = ('authors',) + + def authors_list(self, obj): + return ", ".join([a.name for a in obj.authors.all()]) + + authors_list.short_description = _('authors') + + class EmpruntAdmin(VersionAdmin): list_display = ('media', 'user', 'date_emprunt', 'date_rendu', 'permanencier_emprunt', 'permanencier_rendu_custom') @@ -120,8 +131,10 @@ class JeuAdmin(VersionAdmin): admin_site.register(Auteur, AuteurAdmin) -admin_site.register(Media, MediaAdmin) +admin_site.register(BD, MediaAdmin) admin_site.register(Manga, MediaAdmin) +admin_site.register(CD, CDAdmin) +admin_site.register(Vinyle, CDAdmin) admin_site.register(FutureMedia, FutureMediaAdmin) admin_site.register(Emprunt, EmpruntAdmin) admin_site.register(Jeu, JeuAdmin) diff --git a/media/forms.py b/media/forms.py index f368974..48eff85 100644 --- a/media/forms.py +++ b/media/forms.py @@ -10,7 +10,7 @@ import urllib.request from django.forms import ModelForm from django.utils.translation import gettext_lazy as _ -from .models import Auteur, Media +from .models import Auteur, BD from .scraper import BedetequeScraper @@ -263,5 +263,5 @@ class MediaAdminForm(ModelForm): self.add_error(name, e) class Meta: - model = Media + model = BD fields = '__all__' diff --git a/media/management/commands/split_media_types.py b/media/management/commands/split_media_types.py index b5d7d3c..823009a 100644 --- a/media/management/commands/split_media_types.py +++ b/media/management/commands/split_media_types.py @@ -1,7 +1,9 @@ +from time import sleep + from django.core.management import BaseCommand from media.forms import MediaAdminForm -from media.models import Manga, Media +from media.models import BD, Manga class Command(BaseCommand): @@ -13,7 +15,11 @@ class Command(BaseCommand): def handle(self, *args, **options): converted = 0 - for media in Media.objects.all(): + for media in BD.objects.all(): + if media.pk < 3400: + continue + # We sleep 5 seconds to avoid a ban from Bedetheque + sleep(5) self.stdout.write(str(media)) form = MediaAdminForm(instance=media, data={"isbn": media.isbn, "_isbn": True, }) diff --git a/media/models.py b/media/models.py index 60f7c2f..c7b830a 100644 --- a/media/models.py +++ b/media/models.py @@ -30,7 +30,7 @@ class Auteur(models.Model): ordering = ['name'] -class Media(models.Model): +class BD(models.Model): isbn = ISBNField( _('ISBN'), help_text=_('You may be able to scan it from a bar code.'), @@ -86,8 +86,8 @@ class Media(models.Model): return self.title class Meta: - verbose_name = _("medium") - verbose_name_plural = _("media") + verbose_name = _("BD") + verbose_name_plural = _("BDs") ordering = ['title', 'subtitle'] @@ -140,16 +140,55 @@ class Manga(models.Model): null=True, ) + +class Vinyle(models.Model): + title = models.CharField( + verbose_name=_('title'), + max_length=255, + ) + + side_identifier = models.CharField( + verbose_name=_('side identifier'), + max_length=255, + ) + + authors = models.ManyToManyField( + 'Auteur', + verbose_name=_('authors'), + ) + def __str__(self): - if self.subtitle: - return "{} : {}".format(self.title, self.subtitle) - else: - return self.title + return self.title class Meta: - verbose_name = _("manga") - verbose_name_plural = _("mangas") - ordering = ['title', 'subtitle'] + verbose_name = _("vinyle") + verbose_name_plural = _("vinyles") + ordering = ['title'] + + +class CD(models.Model): + title = models.CharField( + verbose_name=_('title'), + max_length=255, + ) + + side_identifier = models.CharField( + verbose_name=_('side identifier'), + max_length=255, + ) + + authors = models.ManyToManyField( + 'Auteur', + verbose_name=_('authors'), + ) + + def __str__(self): + return self.title + + class Meta: + verbose_name = _("CD") + verbose_name_plural = _("CDs") + ordering = ['title'] class FutureMedia(models.Model): @@ -171,7 +210,7 @@ class FutureMedia(models.Model): class Emprunt(models.Model): media = models.ForeignKey( - 'Media', + 'BD', on_delete=models.PROTECT, ) user = models.ForeignKey( diff --git a/media/serializers.py b/media/serializers.py index d16547c..bbdce2e 100644 --- a/media/serializers.py +++ b/media/serializers.py @@ -1,6 +1,6 @@ from rest_framework import serializers -from .models import Auteur, Emprunt, Jeu, Media +from .models import Auteur, BD, Emprunt, Jeu class AuteurSerializer(serializers.HyperlinkedModelSerializer): @@ -11,7 +11,7 @@ class AuteurSerializer(serializers.HyperlinkedModelSerializer): class MediaSerializer(serializers.HyperlinkedModelSerializer): class Meta: - model = Media + model = BD fields = ['url', 'isbn', 'title', 'subtitle', 'external_url', 'side_identifier', 'authors', 'number_of_pages', 'publish_date'] diff --git a/media/tests/test_templates.py b/media/tests/test_templates.py index 0a1a150..4c2ffbd 100644 --- a/media/tests/test_templates.py +++ b/media/tests/test_templates.py @@ -4,7 +4,7 @@ from django.test import TestCase from django.urls import reverse -from media.models import Auteur, Media +from media.models import Auteur, BD from users.models import User """ @@ -25,12 +25,12 @@ class TemplateTests(TestCase): self.dummy_author = Auteur.objects.create(name="Test author") # Create media - self.dummy_media1 = Media.objects.create( + self.dummy_media1 = BD.objects.create( title="Test media", side_identifier="T M", ) self.dummy_media1.authors.add(self.dummy_author) - self.dummy_media2 = Media.objects.create( + self.dummy_media2 = BD.objects.create( title="Test media bis", side_identifier="T M 2", external_url="https://example.com/", diff --git a/media/views.py b/media/views.py index 3ed0b45..e6ac52f 100644 --- a/media/views.py +++ b/media/views.py @@ -11,7 +11,7 @@ from django.utils.translation import gettext_lazy as _ from rest_framework import viewsets from reversion import revisions as reversion -from .models import Auteur, Emprunt, Jeu, Media +from .models import Auteur, BD, Emprunt, Jeu from .serializers import AuteurSerializer, EmpruntSerializer, \ JeuSerializer, MediaSerializer @@ -57,7 +57,7 @@ class MediaViewSet(viewsets.ModelViewSet): """ API endpoint that allows media to be viewed or edited. """ - queryset = Media.objects.all() + queryset = BD.objects.all() serializer_class = MediaSerializer From 4ab2e9df570ce1f053b5777455463b191ef32b5b Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 18:06:50 +0200 Subject: [PATCH 67/79] Forgot migrations --- media/migrations/0030_auto_20200522_1757.py | 49 +++++++++++++++++++++ media/migrations/0031_auto_20200522_1758.py | 17 +++++++ 2 files changed, 66 insertions(+) create mode 100644 media/migrations/0030_auto_20200522_1757.py create mode 100644 media/migrations/0031_auto_20200522_1758.py diff --git a/media/migrations/0030_auto_20200522_1757.py b/media/migrations/0030_auto_20200522_1757.py new file mode 100644 index 0000000..6e8f84d --- /dev/null +++ b/media/migrations/0030_auto_20200522_1757.py @@ -0,0 +1,49 @@ +# Generated by Django 2.2.10 on 2020-05-22 15:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0029_auto_20200521_1659'), + ] + + operations = [ + migrations.RenameModel( + old_name='Media', + new_name='BD', + ), + migrations.AlterModelOptions( + name='manga', + options={}, + ), + migrations.CreateModel( + name='Vinyle', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255, verbose_name='title')), + ('side_identifier', models.CharField(max_length=255, verbose_name='side identifier')), + ('authors', models.ManyToManyField(to='media.Auteur', verbose_name='authors')), + ], + options={ + 'verbose_name': 'vinyle', + 'verbose_name_plural': 'vinyles', + 'ordering': ['title'], + }, + ), + migrations.CreateModel( + name='CD', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255, verbose_name='title')), + ('side_identifier', models.CharField(max_length=255, verbose_name='side identifier')), + ('authors', models.ManyToManyField(to='media.Auteur', verbose_name='authors')), + ], + options={ + 'verbose_name': 'CD', + 'verbose_name_plural': 'CDs', + 'ordering': ['title'], + }, + ), + ] diff --git a/media/migrations/0031_auto_20200522_1758.py b/media/migrations/0031_auto_20200522_1758.py new file mode 100644 index 0000000..2ff8183 --- /dev/null +++ b/media/migrations/0031_auto_20200522_1758.py @@ -0,0 +1,17 @@ +# Generated by Django 2.2.10 on 2020-05-22 15:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0030_auto_20200522_1757'), + ] + + operations = [ + migrations.AlterModelOptions( + name='bd', + options={'ordering': ['title', 'subtitle'], 'verbose_name': 'BD', 'verbose_name_plural': 'BDs'}, + ), + ] From 20cb710af5994191a1e67f213bb7f417bf816e0b Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 21:08:44 +0200 Subject: [PATCH 68/79] Romans were missing --- media/admin.py | 3 +- media/forms.py | 1 + media/migrations/0032_auto_20200522_2107.py | 38 ++++++++++++ media/models.py | 66 +++++++++++++++++++++ media/tests/test_templates.py | 26 ++++---- 5 files changed, 120 insertions(+), 14 deletions(-) create mode 100644 media/migrations/0032_auto_20200522_2107.py diff --git a/media/admin.py b/media/admin.py index bd92c6b..71744fd 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga, Vinyle +from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga, Roman, Vinyle class AuteurAdmin(VersionAdmin): @@ -133,6 +133,7 @@ class JeuAdmin(VersionAdmin): admin_site.register(Auteur, AuteurAdmin) admin_site.register(BD, MediaAdmin) admin_site.register(Manga, MediaAdmin) +admin_site.register(Roman, MediaAdmin) admin_site.register(CD, CDAdmin) admin_site.register(Vinyle, CDAdmin) admin_site.register(FutureMedia, FutureMediaAdmin) diff --git a/media/forms.py b/media/forms.py index 48eff85..08e766c 100644 --- a/media/forms.py +++ b/media/forms.py @@ -166,6 +166,7 @@ class MediaAdminForm(ModelForm): if "_isbn_addanother" in self.data: self.data = self.data.copy() self.data['_addanother'] = 42 + self.request.POST = self.data if isbn: # ISBN is present, try with bedeteque scrap_result = self.download_data_bedeteque(isbn) diff --git a/media/migrations/0032_auto_20200522_2107.py b/media/migrations/0032_auto_20200522_2107.py new file mode 100644 index 0000000..30073a0 --- /dev/null +++ b/media/migrations/0032_auto_20200522_2107.py @@ -0,0 +1,38 @@ +# Generated by Django 2.2.10 on 2020-05-22 19:07 + +from django.db import migrations, models +import media.fields +import media.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0031_auto_20200522_1758'), + ] + + operations = [ + migrations.AlterModelOptions( + name='manga', + options={'ordering': ['title'], 'verbose_name': 'manga', 'verbose_name_plural': 'mangas'}, + ), + migrations.CreateModel( + name='Roman', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('isbn', media.fields.ISBNField(blank=True, help_text='You may be able to scan it from a bar code.', max_length=28, null=True, unique=True, validators=[media.validators.isbn_validator], verbose_name='ISBN')), + ('title', models.CharField(max_length=255, verbose_name='title')), + ('subtitle', models.CharField(blank=True, max_length=255, null=True, verbose_name='subtitle')), + ('external_url', models.URLField(blank=True, null=True, verbose_name='external URL')), + ('side_identifier', models.CharField(max_length=255, verbose_name='side identifier')), + ('number_of_pages', models.PositiveIntegerField(blank=True, null=True, verbose_name='number of pages')), + ('publish_date', models.DateField(blank=True, null=True, verbose_name='publish date')), + ('authors', models.ManyToManyField(to='media.Auteur', verbose_name='authors')), + ], + options={ + 'verbose_name': 'roman', + 'verbose_name_plural': 'romans', + 'ordering': ['title', 'subtitle'], + }, + ), + ] diff --git a/media/models.py b/media/models.py index c7b830a..ade119c 100644 --- a/media/models.py +++ b/media/models.py @@ -140,6 +140,72 @@ class Manga(models.Model): null=True, ) + def __str__(self): + return self.title + + class Meta: + verbose_name = _("manga") + verbose_name_plural = _("mangas") + ordering = ['title'] + + +class Roman(models.Model): + isbn = ISBNField( + _('ISBN'), + help_text=_('You may be able to scan it from a bar code.'), + unique=True, + blank=True, + null=True, + ) + + title = models.CharField( + verbose_name=_('title'), + max_length=255, + ) + + subtitle = models.CharField( + verbose_name=_('subtitle'), + max_length=255, + blank=True, + null=True, + ) + + external_url = models.URLField( + verbose_name=_('external URL'), + blank=True, + null=True, + ) + + side_identifier = models.CharField( + verbose_name=_('side identifier'), + max_length=255, + ) + + authors = models.ManyToManyField( + 'Auteur', + verbose_name=_('authors'), + ) + + number_of_pages = models.PositiveIntegerField( + verbose_name=_('number of pages'), + blank=True, + null=True, + ) + + publish_date = models.DateField( + verbose_name=_('publish date'), + blank=True, + null=True, + ) + + def __str__(self): + return self.title + + class Meta: + verbose_name = _("roman") + verbose_name_plural = _("romans") + ordering = ['title', 'subtitle'] + class Vinyle(models.Model): title = models.CharField( diff --git a/media/tests/test_templates.py b/media/tests/test_templates.py index 4c2ffbd..c103359 100644 --- a/media/tests/test_templates.py +++ b/media/tests/test_templates.py @@ -25,41 +25,41 @@ class TemplateTests(TestCase): self.dummy_author = Auteur.objects.create(name="Test author") # Create media - self.dummy_media1 = BD.objects.create( + self.dummy_bd1 = BD.objects.create( title="Test media", side_identifier="T M", ) - self.dummy_media1.authors.add(self.dummy_author) - self.dummy_media2 = BD.objects.create( + self.dummy_bd1.authors.add(self.dummy_author) + self.dummy_bd2 = BD.objects.create( title="Test media bis", side_identifier="T M 2", external_url="https://example.com/", ) - self.dummy_media2.authors.add(self.dummy_author) + self.dummy_bd2.authors.add(self.dummy_author) - def test_media_media_changelist(self): - response = self.client.get(reverse('admin:media_media_changelist')) + def test_bd_bd_changelist(self): + response = self.client.get(reverse('admin:media_bd_changelist')) self.assertEqual(response.status_code, 200) - def test_media_media_add(self): - response = self.client.get(reverse('admin:media_media_add')) + def test_bd_bd_add(self): + response = self.client.get(reverse('admin:media_bd_add')) self.assertEqual(response.status_code, 200) - def test_media_isbn_download(self): + def test_bd_isbn_download(self): data = { '_isbn': True, 'isbn': "0316358525", } response = self.client.post(reverse( - 'admin:media_media_change', - args=[self.dummy_media1.id], + 'admin:media_bd_change', + args=[self.dummy_bd1.id], ), data=data) self.assertEqual(response.status_code, 302) - def test_media_emprunt_changelist(self): + def test_bd_emprunt_changelist(self): response = self.client.get(reverse('admin:media_emprunt_changelist')) self.assertEqual(response.status_code, 200) - def test_media_emprunt_add(self): + def test_bd_emprunt_add(self): response = self.client.get(reverse('admin:media_emprunt_add')) self.assertEqual(response.status_code, 200) From bed5912f545a3e7d731ce439cb57c34063a1be2f Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 21:37:02 +0200 Subject: [PATCH 69/79] Cat ISBN list to future medias --- media/management/commands/import_isbn.py | 94 +++++++++++++++++++++++ media/migrations/0033_futuremedia_type.py | 19 +++++ media/models.py | 10 +++ 3 files changed, 123 insertions(+) create mode 100644 media/management/commands/import_isbn.py create mode 100644 media/migrations/0033_futuremedia_type.py diff --git a/media/management/commands/import_isbn.py b/media/management/commands/import_isbn.py new file mode 100644 index 0000000..1a95a95 --- /dev/null +++ b/media/management/commands/import_isbn.py @@ -0,0 +1,94 @@ +from argparse import FileType +from sys import stdin + +from django.core.exceptions import ValidationError +from django.core.management import BaseCommand + +from media.models import BD, FutureMedia, Manga, Roman +from media.validators import isbn_validator + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--media-type', + type=str, + default='bd', + choices=[ + 'bd', + 'manga', + 'roman', + ], + help="Type of media to be " + "imported.") + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="ISBN to be imported.") + + def handle(self, *args, **options): + print(options) + type_str = options["media_type"] + + media_classes = [BD, Manga, Roman, FutureMedia] + + file = options["input"] + isbns = [] + for line in file: + isbns.append(line[:-1]) + + print("Registering", len(isbns), "ISBN") + + imported = 0 + not_imported = [] + + for isbn in isbns: + if not isbn: + continue + + try: + if not isbn_validator(isbn): + raise ValidationError( + "This ISBN is invalid for an unknown reason") + except ValidationError as e: + self.stderr.write(self.style.ERROR( + "The following ISBN is invalid:" + " {isbn}, reason: {reason}. Ignoring...".format( + isbn=isbn, reason=e.message))) + + isbn_exists = False + for cl in media_classes: + if cl.objects.filter(isbn=isbn).exists(): + isbn_exists = True + medium = cl.objects.get(isbn=isbn) + self.stderr.write(self.style.WARNING( + ("Warning: ISBN {isbn} already exists, and is " + + "registered as type {type}: {name}. Ignoring...") + .format(isbn=isbn, + name=str(medium), + type=str(cl._meta.verbose_name)))) + not_imported.append(medium) + break + + if isbn_exists: + continue + + FutureMedia.objects.create(isbn=isbn, type=type_str) + self.stdout.write(self.style.SUCCESS("ISBN {isbn} imported" + .format(isbn=isbn))) + imported += 1 + + self.stdout.write(self.style.SUCCESS("{count} media imported" + .format(count=imported))) + + with open('not_imported_media.csv', 'w') as f: + f.write("isbn|type|title\n") + for medium in not_imported: + if not hasattr(medium, 'title') or not medium.title: + medium.title = '' + f.write(medium.isbn + "|" + + str(medium._meta.verbose_name) + + "|" + medium.title + "\n") + + self.stderr.write(self.style.WARNING(("{count} media already " + + "imported").format( + count=len(not_imported)))) diff --git a/media/migrations/0033_futuremedia_type.py b/media/migrations/0033_futuremedia_type.py new file mode 100644 index 0000000..b963e06 --- /dev/null +++ b/media/migrations/0033_futuremedia_type.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.10 on 2020-05-22 19:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0032_auto_20200522_2107'), + ] + + operations = [ + migrations.AddField( + model_name='futuremedia', + name='type', + field=models.CharField(choices=[('bd', 'BD'), ('manga', 'Manga'), ('roman', 'Roman')], default='bd', max_length=8, verbose_name='type'), + preserve_default=False, + ), + ] diff --git a/media/models.py b/media/models.py index ade119c..428e601 100644 --- a/media/models.py +++ b/media/models.py @@ -266,6 +266,16 @@ class FutureMedia(models.Model): null=True, ) + type = models.CharField( + _('type'), + choices=[ + ('bd', _('BD')), + ('manga', _('Manga')), + ('roman', _('Roman')), + ], + max_length=8, + ) + class Meta: verbose_name = _("future medium") verbose_name_plural = _("future media") From 50f3cf39c1b00d2c58c2dc3c980a8c01902b7e08 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 21:49:09 +0200 Subject: [PATCH 70/79] Import future media --- media/admin.py | 3 +- .../commands/import_future_media.py | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 media/management/commands/import_future_media.py diff --git a/media/admin.py b/media/admin.py index 71744fd..65f42c0 100644 --- a/media/admin.py +++ b/media/admin.py @@ -9,7 +9,8 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm -from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga, Roman, Vinyle +from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga,\ + Roman, Vinyle class AuteurAdmin(VersionAdmin): diff --git a/media/management/commands/import_future_media.py b/media/management/commands/import_future_media.py new file mode 100644 index 0000000..bbd5673 --- /dev/null +++ b/media/management/commands/import_future_media.py @@ -0,0 +1,47 @@ +from time import sleep + +from django.core.exceptions import ValidationError +from django.core.management import BaseCommand + +from media.forms import MediaAdminForm +from media.models import BD, FutureMedia, Manga, Roman + + +class Command(BaseCommand): + def handle(self, *args, **options): + for future_medium in FutureMedia.objects.all(): + isbn = future_medium.isbn + type_str = future_medium.type + if type_str == 'bd': + cl = BD + elif type_str == 'manga': + cl = Manga + elif type_str == 'roman': + cl = Roman + else: + self.stderr.write(self.style.WARNING( + "Unknown medium type: {type}. Ignoring..." + .format(type=type_str))) + continue + + if cl.objects.filter(isbn=isbn).exists(): + self.stderr.write(self.style.WARNING( + "ISBN {isbn} already exists".format(isbn=isbn) + )) + + form = MediaAdminForm(instance=cl(), + data={"isbn": isbn, "_isbn": True, }) + # Don't DDOS any website + sleep(5) + + try: + form.full_clean() + form.save() + future_medium.delete() + self.stdout.write(self.style.SUCCESS( + "Medium with ISBN {isbn} successfully imported" + .format(isbn=isbn))) + except ValidationError as e: + self.stderr.write(self.style.WARNING( + "An error occured while importing ISBN {isbn}: {error}" + .format(isbn=isbn, error=e.message))) From d88fccb51d9f25db481930bde7d6a8b96447e2f8 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 22:15:37 +0200 Subject: [PATCH 71/79] Import vinyles --- .../commands/import_future_media.py | 4 +- media/management/commands/import_isbn.py | 1 - media/management/commands/import_vinyles.py | 51 +++++++++++++++++++ 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 media/management/commands/import_vinyles.py diff --git a/media/management/commands/import_future_media.py b/media/management/commands/import_future_media.py index bbd5673..3e39022 100644 --- a/media/management/commands/import_future_media.py +++ b/media/management/commands/import_future_media.py @@ -41,7 +41,7 @@ class Command(BaseCommand): self.stdout.write(self.style.SUCCESS( "Medium with ISBN {isbn} successfully imported" .format(isbn=isbn))) - except ValidationError as e: + except (ValidationError, ValueError) as e: self.stderr.write(self.style.WARNING( "An error occured while importing ISBN {isbn}: {error}" - .format(isbn=isbn, error=e.message))) + .format(isbn=isbn, error=str(e)))) diff --git a/media/management/commands/import_isbn.py b/media/management/commands/import_isbn.py index 1a95a95..9db98cb 100644 --- a/media/management/commands/import_isbn.py +++ b/media/management/commands/import_isbn.py @@ -26,7 +26,6 @@ class Command(BaseCommand): help="ISBN to be imported.") def handle(self, *args, **options): - print(options) type_str = options["media_type"] media_classes = [BD, Manga, Roman, FutureMedia] diff --git a/media/management/commands/import_vinyles.py b/media/management/commands/import_vinyles.py new file mode 100644 index 0000000..18343dd --- /dev/null +++ b/media/management/commands/import_vinyles.py @@ -0,0 +1,51 @@ +from argparse import FileType +from sys import stdin + +from django.core.management import BaseCommand + +from media.models import Auteur, Vinyle + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="ISBN to be imported.") + + def handle(self, *args, **options): + file = options["input"] + vinyles = [] + for line in file: + vinyles.append(line[:-1].split('|', 2)) + + print("Registering", len(vinyles), "vinyles") + + imported = 0 + + for vinyle in vinyles: + if len(vinyle) != 3: + continue + + side = vinyle[0] + title = vinyle[1] + authors_str = vinyle[2].split('|') + authors = [Auteur.objects.get_or_create(name=author)[0] + for author in authors_str] + vinyle, created = Vinyle.objects.get_or_create( + title=title, + side_identifier=side, + ) + vinyle.authors.set(authors) + vinyle.save() + + if not created: + self.stderr.write(self.style.WARNING( + "One vinyle was already imported. Skipping...")) + else: + self.stdout.write(self.style.SUCCESS( + "Vinyle imported")) + imported += 1 + + self.stdout.write(self.style.SUCCESS( + "{count} vinyles imported".format(count=imported))) From 02b81016b89ca39136d04dec5b21feadc07222bb Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Fri, 22 May 2020 22:17:17 +0200 Subject: [PATCH 72/79] Import CDs --- media/management/commands/import_cds.py | 51 +++++++++++++++++++++ media/management/commands/import_vinyles.py | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 media/management/commands/import_cds.py diff --git a/media/management/commands/import_cds.py b/media/management/commands/import_cds.py new file mode 100644 index 0000000..67c60bc --- /dev/null +++ b/media/management/commands/import_cds.py @@ -0,0 +1,51 @@ +from argparse import FileType +from sys import stdin + +from django.core.management import BaseCommand + +from media.models import Auteur, Vinyle + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="CD to be imported.") + + def handle(self, *args, **options): + file = options["input"] + cds = [] + for line in file: + cds.append(line[:-1].split('|', 2)) + + print("Registering", len(cds), "CDs") + + imported = 0 + + for cd in cds: + if len(cd) != 3: + continue + + side = cd[0] + title = cd[1] + authors_str = cd[2].split('|') + authors = [Auteur.objects.get_or_create(name=author)[0] + for author in authors_str] + cd, created = Vinyle.objects.get_or_create( + title=title, + side_identifier=side, + ) + cd.authors.set(authors) + cd.save() + + if not created: + self.stderr.write(self.style.WARNING( + "One CD was already imported. Skipping...")) + else: + self.stdout.write(self.style.SUCCESS( + "CD imported")) + imported += 1 + + self.stdout.write(self.style.SUCCESS( + "{count} CDs imported".format(count=imported))) diff --git a/media/management/commands/import_vinyles.py b/media/management/commands/import_vinyles.py index 18343dd..8fc6722 100644 --- a/media/management/commands/import_vinyles.py +++ b/media/management/commands/import_vinyles.py @@ -11,7 +11,7 @@ class Command(BaseCommand): parser.add_argument('input', nargs='?', type=FileType('r'), default=stdin, - help="ISBN to be imported.") + help="Vinyle to be imported.") def handle(self, *args, **options): file = options["input"] From e2d4a80dba3195b79b3997bdd1686096cb89dff0 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sat, 23 May 2020 14:23:39 +0200 Subject: [PATCH 73/79] Store RPM in vinyles --- media/management/commands/import_vinyles.py | 7 +++++++ media/models.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/media/management/commands/import_vinyles.py b/media/management/commands/import_vinyles.py index 8fc6722..2c189a4 100644 --- a/media/management/commands/import_vinyles.py +++ b/media/management/commands/import_vinyles.py @@ -13,7 +13,13 @@ class Command(BaseCommand): default=stdin, help="Vinyle to be imported.") + parser.add_argument('--rpm', + type=int, + default=45, + help="RPM of the imported vinyles.") + def handle(self, *args, **options): + rpm = options["rpm"] file = options["input"] vinyles = [] for line in file: @@ -35,6 +41,7 @@ class Command(BaseCommand): vinyle, created = Vinyle.objects.get_or_create( title=title, side_identifier=side, + rp=rpm, ) vinyle.authors.set(authors) vinyle.save() diff --git a/media/models.py b/media/models.py index 428e601..3517445 100644 --- a/media/models.py +++ b/media/models.py @@ -218,6 +218,14 @@ class Vinyle(models.Model): max_length=255, ) + rpm = models.PositiveIntegerField( + verbose_name=_('rounds per minute'), + choices=[ + (33, _('33 RPM')), + (45, _('45 RPM')), + ], + ) + authors = models.ManyToManyField( 'Auteur', verbose_name=_('authors'), From 28eac94312b70e318196d0aeaf9a6ea242196df7 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sat, 23 May 2020 14:31:22 +0200 Subject: [PATCH 74/79] Store RPM in vinyles --- media/migrations/0034_vinyle_rpm.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 media/migrations/0034_vinyle_rpm.py diff --git a/media/migrations/0034_vinyle_rpm.py b/media/migrations/0034_vinyle_rpm.py new file mode 100644 index 0000000..b17a642 --- /dev/null +++ b/media/migrations/0034_vinyle_rpm.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.10 on 2020-05-23 12:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0033_futuremedia_type'), + ] + + operations = [ + migrations.AddField( + model_name='vinyle', + name='rpm', + field=models.PositiveIntegerField(choices=[(33, '33 RPM'), (45, '45 RPM')], default=45, verbose_name='rounds per minute'), + preserve_default=False, + ), + ] From 3977ab9ec30a7908de1bafc1cba69bb69a92fda0 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sat, 23 May 2020 16:42:59 +0200 Subject: [PATCH 75/79] Fix CD import --- media/management/commands/import_cds.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/media/management/commands/import_cds.py b/media/management/commands/import_cds.py index 67c60bc..d783a12 100644 --- a/media/management/commands/import_cds.py +++ b/media/management/commands/import_cds.py @@ -3,7 +3,7 @@ from sys import stdin from django.core.management import BaseCommand -from media.models import Auteur, Vinyle +from media.models import Auteur, CD class Command(BaseCommand): @@ -27,12 +27,12 @@ class Command(BaseCommand): if len(cd) != 3: continue - side = cd[0] - title = cd[1] + title = cd[0] + side = cd[1] authors_str = cd[2].split('|') authors = [Auteur.objects.get_or_create(name=author)[0] for author in authors_str] - cd, created = Vinyle.objects.get_or_create( + cd, created = CD.objects.get_or_create( title=title, side_identifier=side, ) From 963ff25506ece3025e3a7fa8d6da2770d5ec48e0 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 24 May 2020 15:37:57 +0200 Subject: [PATCH 76/79] Add revues --- media/admin.py | 8 ++- media/management/commands/import_marvel.py | 50 ++++++++++++++++++ media/management/commands/import_revues.py | 58 +++++++++++++++++++++ media/migrations/0035_revue.py | 29 +++++++++++ media/migrations/0036_auto_20200524_1500.py | 18 +++++++ media/migrations/0037_revue_double.py | 18 +++++++ media/models.py | 45 ++++++++++++++++ 7 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 media/management/commands/import_marvel.py create mode 100644 media/management/commands/import_revues.py create mode 100644 media/migrations/0035_revue.py create mode 100644 media/migrations/0036_auto_20200524_1500.py create mode 100644 media/migrations/0037_revue_double.py diff --git a/media/admin.py b/media/admin.py index 65f42c0..abe0f86 100644 --- a/media/admin.py +++ b/media/admin.py @@ -10,7 +10,7 @@ from reversion.admin import VersionAdmin from med.admin import admin_site from .forms import MediaAdminForm from .models import Auteur, BD, CD, Emprunt, FutureMedia, Jeu, Manga,\ - Roman, Vinyle + Revue, Roman, Vinyle class AuteurAdmin(VersionAdmin): @@ -88,6 +88,11 @@ class CDAdmin(VersionAdmin): authors_list.short_description = _('authors') +class RevueAdmin(VersionAdmin): + list_display = ('__str__', 'number', 'year', 'month', 'day', 'double',) + search_fields = ('title', 'number', 'year',) + + class EmpruntAdmin(VersionAdmin): list_display = ('media', 'user', 'date_emprunt', 'date_rendu', 'permanencier_emprunt', 'permanencier_rendu_custom') @@ -137,6 +142,7 @@ admin_site.register(Manga, MediaAdmin) admin_site.register(Roman, MediaAdmin) admin_site.register(CD, CDAdmin) admin_site.register(Vinyle, CDAdmin) +admin_site.register(Revue, RevueAdmin) admin_site.register(FutureMedia, FutureMediaAdmin) admin_site.register(Emprunt, EmpruntAdmin) admin_site.register(Jeu, JeuAdmin) diff --git a/media/management/commands/import_marvel.py b/media/management/commands/import_marvel.py new file mode 100644 index 0000000..69d5b1c --- /dev/null +++ b/media/management/commands/import_marvel.py @@ -0,0 +1,50 @@ +from argparse import FileType +from sys import stdin + +from django.core.management import BaseCommand +from media.models import Auteur, BD + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="Marvel comic to be imported.") + + def handle(self, *args, **options): + file = options["input"] + revues = [] + for line in file: + revues.append(line[:-1].split('|', 2)) + + print("Registering", len(revues), "Marvel comics") + + imported = 0 + + for revue in revues: + if len(revue) != 3: + continue + + title = revue[0] + number = revue[1] + authors = [Auteur.objects.get_or_create(name=n)[0] + for n in revue[2].split('|')] + bd = BD.objects.create( + title=title, + subtitle=number, + side_identifier="{:.3} {:.3} {:0>2}" + .format(authors[0].name.upper(), + title.upper(), + number), + ) + + bd.authors.set(authors) + bd.save() + + self.stdout.write(self.style.SUCCESS( + "Comic imported")) + imported += 1 + + self.stdout.write(self.style.SUCCESS( + "{count} comics imported".format(count=imported))) diff --git a/media/management/commands/import_revues.py b/media/management/commands/import_revues.py new file mode 100644 index 0000000..6df08b3 --- /dev/null +++ b/media/management/commands/import_revues.py @@ -0,0 +1,58 @@ +from argparse import FileType +from sys import stdin + +from django.core.management import BaseCommand +from media.models import Revue + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="Revues to be imported.") + + def handle(self, *args, **options): + file = options["input"] + revues = [] + for line in file: + revues.append(line[:-1].split('|')) + + print("Registering", len(revues), "revues") + + imported = 0 + + for revue in revues: + if len(revue) != 5: + continue + + title = revue[0] + number = revue[1] + day = revue[2] + if not day: + day = None + month = revue[3] + if not month: + month = None + year = revue[4] + if not year: + year = None + revue, created = Revue.objects.get_or_create( + title=title, + number=number.replace('*', ''), + year=year, + month=month, + day=day, + double=number.endswith('*'), + ) + + if not created: + self.stderr.write(self.style.WARNING( + "One revue was already imported. Skipping...")) + else: + self.stdout.write(self.style.SUCCESS( + "Revue imported")) + imported += 1 + + self.stdout.write(self.style.SUCCESS( + "{count} revues imported".format(count=imported))) diff --git a/media/migrations/0035_revue.py b/media/migrations/0035_revue.py new file mode 100644 index 0000000..0da2b3f --- /dev/null +++ b/media/migrations/0035_revue.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.10 on 2020-05-24 12:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0034_vinyle_rpm'), + ] + + operations = [ + migrations.CreateModel( + name='Revue', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=255, verbose_name='title')), + ('number', models.PositiveIntegerField(verbose_name='number')), + ('year', models.PositiveIntegerField(verbose_name='year')), + ('month', models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='month')), + ('day', models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='day')), + ], + options={ + 'verbose_name': 'revue', + 'verbose_name_plural': 'revues', + 'ordering': ['title', 'number'], + }, + ), + ] diff --git a/media/migrations/0036_auto_20200524_1500.py b/media/migrations/0036_auto_20200524_1500.py new file mode 100644 index 0000000..064fa62 --- /dev/null +++ b/media/migrations/0036_auto_20200524_1500.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.10 on 2020-05-24 13:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0035_revue'), + ] + + operations = [ + migrations.AlterField( + model_name='revue', + name='year', + field=models.PositiveIntegerField(blank=True, default=None, null=True, verbose_name='year'), + ), + ] diff --git a/media/migrations/0037_revue_double.py b/media/migrations/0037_revue_double.py new file mode 100644 index 0000000..8c67fbc --- /dev/null +++ b/media/migrations/0037_revue_double.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.10 on 2020-05-24 13:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('media', '0036_auto_20200524_1500'), + ] + + operations = [ + migrations.AddField( + model_name='revue', + name='double', + field=models.BooleanField(default=False, verbose_name='double'), + ), + ] diff --git a/media/models.py b/media/models.py index 3517445..d6df7ba 100644 --- a/media/models.py +++ b/media/models.py @@ -265,6 +265,51 @@ class CD(models.Model): ordering = ['title'] +class Revue(models.Model): + title = models.CharField( + verbose_name=_('title'), + max_length=255, + ) + + number = models.PositiveIntegerField( + verbose_name=_('number'), + ) + + year = models.PositiveIntegerField( + verbose_name=_('year'), + null=True, + blank=True, + default=None, + ) + + month = models.PositiveIntegerField( + verbose_name=_('month'), + null=True, + blank=True, + default=None, + ) + + day = models.PositiveIntegerField( + verbose_name=_('day'), + null=True, + blank=True, + default=None, + ) + + double = models.BooleanField( + verbose_name=_('double'), + default=False, + ) + + def __str__(self): + return self.title + " n°" + str(self.number) + + class Meta: + verbose_name = _("revue") + verbose_name_plural = _("revues") + ordering = ['title', 'number'] + + class FutureMedia(models.Model): isbn = ISBNField( _('ISBN'), From 44abcaf20202c15ebd1b9bb3d6b725b9ee3435f9 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 24 May 2020 16:01:05 +0200 Subject: [PATCH 77/79] Import romans without any ISBN --- .../commands/import_no_isbn_roman.py | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 media/management/commands/import_no_isbn_roman.py diff --git a/media/management/commands/import_no_isbn_roman.py b/media/management/commands/import_no_isbn_roman.py new file mode 100644 index 0000000..5e822fc --- /dev/null +++ b/media/management/commands/import_no_isbn_roman.py @@ -0,0 +1,65 @@ +import re +import unicodedata +from argparse import FileType +from sys import stdin + +from django.core.management import BaseCommand + +from media.models import Auteur, Roman + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('input', nargs='?', + type=FileType('r'), + default=stdin, + help="Revues to be imported.") + + def handle(self, *args, **options): + file = options["input"] + romans = [] + for line in file: + romans.append(line[:-1].split('|')) + + print("Registering", len(romans), "romans") + + imported = 0 + + for book in romans: + if len(book) != 2: + continue + + title = book[1] + title_normalized = title.upper() + title_normalized = title_normalized.replace('’', '\'') + title_normalized = ''.join( + char + for char in unicodedata.normalize( + 'NFKD', title_normalized.casefold()) + if all(not unicodedata.category(char).startswith(cat) + for cat in {'M', 'P', 'Z', 'C'}) or char == ' ' + ).casefold().upper() + title_normalized = re.sub(r'^LE ', '', title_normalized) + title_normalized = re.sub(r'^LA ', '', title_normalized) + title_normalized = re.sub(r'^LES ', '', title_normalized) + title_normalized = re.sub(r'^L\'', '', title_normalized) + title_normalized = re.sub(r'^THE ', '', title_normalized) + title_normalized = re.sub(r'Œ', 'OE', title_normalized) + title_normalized = title_normalized.replace(' ', '') + authors = [Auteur.objects.get_or_create(name=n)[0] + for n in book[0].split(';')] + side_identifier = "{:.3} {:.3}" \ + .format(authors[0].name.upper(), title_normalized, ) + roman = Roman.objects.create( + title=title, + side_identifier=side_identifier, + ) + roman.authors.set(authors) + roman.save() + + self.stdout.write(self.style.SUCCESS( + "Roman imported")) + imported += 1 + + self.stdout.write(self.style.SUCCESS( + "{count} romans imported".format(count=imported))) From e63d8630ccf2a95d9907f70106df90d927438099 Mon Sep 17 00:00:00 2001 From: Yohann D'ANELLO Date: Sun, 24 May 2020 17:47:37 +0200 Subject: [PATCH 78/79] Import 45 RPM vinyles --- media/admin.py | 13 ++++++++++++- media/management/commands/import_vinyles.py | 7 ++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/media/admin.py b/media/admin.py index abe0f86..3ecbab2 100644 --- a/media/admin.py +++ b/media/admin.py @@ -88,6 +88,17 @@ class CDAdmin(VersionAdmin): authors_list.short_description = _('authors') +class VinyleAdmin(VersionAdmin): + list_display = ('title', 'authors_list', 'side_identifier', 'rpm',) + search_fields = ('title', 'authors__name', 'side_identifier', 'rpm',) + autocomplete_fields = ('authors',) + + def authors_list(self, obj): + return ", ".join([a.name for a in obj.authors.all()]) + + authors_list.short_description = _('authors') + + class RevueAdmin(VersionAdmin): list_display = ('__str__', 'number', 'year', 'month', 'day', 'double',) search_fields = ('title', 'number', 'year',) @@ -141,7 +152,7 @@ admin_site.register(BD, MediaAdmin) admin_site.register(Manga, MediaAdmin) admin_site.register(Roman, MediaAdmin) admin_site.register(CD, CDAdmin) -admin_site.register(Vinyle, CDAdmin) +admin_site.register(Vinyle, VinyleAdmin) admin_site.register(Revue, RevueAdmin) admin_site.register(FutureMedia, FutureMediaAdmin) admin_site.register(Emprunt, EmpruntAdmin) diff --git a/media/management/commands/import_vinyles.py b/media/management/commands/import_vinyles.py index 2c189a4..dc64372 100644 --- a/media/management/commands/import_vinyles.py +++ b/media/management/commands/import_vinyles.py @@ -34,14 +34,15 @@ class Command(BaseCommand): continue side = vinyle[0] - title = vinyle[1] - authors_str = vinyle[2].split('|') + title = vinyle[1 if rpm == 33 else 2] + authors_str = vinyle[2 if rpm == 33 else 1]\ + .split('|' if rpm == 33 else ';') authors = [Auteur.objects.get_or_create(name=author)[0] for author in authors_str] vinyle, created = Vinyle.objects.get_or_create( title=title, side_identifier=side, - rp=rpm, + rpm=rpm, ) vinyle.authors.set(authors) vinyle.save() From 0147c5b42c68fc56e95824b85162081335aae77f Mon Sep 17 00:00:00 2001 From: Alexandre Iooss Date: Thu, 13 Aug 2020 08:18:15 +0200 Subject: [PATCH 79/79] Revert CAS to make it works on Zamok --- med/settings.py | 7 ------- med/settings_local.example.py | 4 ---- med/urls.py | 6 ------ 3 files changed, 17 deletions(-) diff --git a/med/settings.py b/med/settings.py index 7e79e27..b9b08cd 100644 --- a/med/settings.py +++ b/med/settings.py @@ -35,7 +35,6 @@ INSTALLED_APPS = [ # External apps 'reversion', 'rest_framework', - 'cas', # Django contrib 'django.contrib.admin', @@ -65,14 +64,8 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.contrib.sites.middleware.CurrentSiteMiddleware', - # 'cas.middleware.CASMiddleware', ] -AUTHENTICATION_BACKENDS = ( - 'django.contrib.auth.backends.ModelBackend', - 'cas.backends.CASBackend', -) - ROOT_URLCONF = 'med.urls' TEMPLATES = [ diff --git a/med/settings_local.example.py b/med/settings_local.example.py index c2006a2..51fb051 100644 --- a/med/settings_local.example.py +++ b/med/settings_local.example.py @@ -16,10 +16,6 @@ EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' SERVER_EMAIL = 'club-med@crans.org' -CAS_SERVER_URL = "https://note.crans.org/cas/" -CAS_LOGOUT_COMPLETELY = True -CAS_PROVIDE_URL_TO_LOGOUT = True - # Security settings SECURE_CONTENT_TYPE_NOSNIFF = False SECURE_BROWSER_XSS_FILTER = False diff --git a/med/urls.py b/med/urls.py index ad4bff8..51e3f3d 100644 --- a/med/urls.py +++ b/med/urls.py @@ -2,7 +2,6 @@ # Copyright (C) 2017-2019 by BDE ENS Paris-Saclay # SPDX-License-Identifier: GPL-3.0-or-later -from cas import views as cas_views from django.contrib.auth.decorators import login_required from django.contrib.auth.views import PasswordResetView from django.urls import include, path @@ -42,9 +41,4 @@ urlpatterns = [ path('accounts/profile/', RedirectView.as_view(pattern_name='index')), path('database/doc/', include('django.contrib.admindocs.urls')), path('database/', admin_site.urls), - - # Include CAS authentication - # TODO Uncomment when NK20 will be ready - # path('admin/login/', cas_views.login, name='login'), - # path('admin/logout/', cas_views.logout, name='logout'), ]