337 lines
13 KiB
Python
337 lines
13 KiB
Python
# -*- mode: python; coding: utf-8 -*-
|
||
# Copyright (C) 2017-2020 by BDE ENS Paris-Saclay
|
||
# SPDX-License-Identifier: GPL-3.0-or-later
|
||
|
||
import json
|
||
import os
|
||
import re
|
||
import unicodedata
|
||
from urllib.error import HTTPError
|
||
import urllib.request
|
||
|
||
from django.db.models import QuerySet
|
||
from django.forms import ModelForm
|
||
from django.utils.translation import gettext_lazy as _
|
||
|
||
from .models import Auteur, BD
|
||
from .scraper import BedetequeScraper
|
||
|
||
|
||
def generate_side_identifier(title, authors, subtitle=None):
|
||
if isinstance(authors, QuerySet):
|
||
authors = list(authors)
|
||
|
||
title_normalized = title.upper()
|
||
title_normalized = title_normalized.replace('’', '\'')
|
||
title_normalized = re.sub(r'^DE ', '', title_normalized)
|
||
title_normalized = re.sub(r'^DES ', '', title_normalized)
|
||
title_normalized = re.sub(r'^LE ', '', title_normalized)
|
||
title_normalized = re.sub(r'^LA ', '', title_normalized)
|
||
title_normalized = re.sub(r'^LES ', '', title_normalized)
|
||
title_normalized = re.sub(r'^L\'', '', title_normalized)
|
||
title_normalized = re.sub(r'^UN ', '', title_normalized)
|
||
title_normalized = re.sub(r'^UNE ', '', title_normalized)
|
||
title_normalized = re.sub(r'^THE ', '', title_normalized)
|
||
title_normalized = re.sub(r'Œ', 'OE', title_normalized)
|
||
title_normalized = title_normalized.replace(' ', '')
|
||
title_normalized = ''.join(
|
||
char
|
||
for char in unicodedata.normalize(
|
||
'NFKD', title_normalized.casefold())
|
||
if all(not unicodedata.category(char).startswith(cat)
|
||
for cat in {'M', 'P', 'Z', 'C'}) or char == ' '
|
||
).casefold().upper()
|
||
title_normalized = re.sub("[^A-Z0-9$]", "", title_normalized)
|
||
authors = authors.copy()
|
||
|
||
def sort(author):
|
||
return "{:042d}".format(-author.note) + author.name.split(" ")[-1] + ".{:042d}".format(author.pk)
|
||
|
||
authors.sort(key=sort)
|
||
primary_author = authors[0]
|
||
author_name = primary_author.name.upper()
|
||
if ',' not in author_name and ' ' in author_name:
|
||
author_name = author_name.split(' ')[-1]
|
||
author_name = ''.join(
|
||
char for char in unicodedata.normalize('NFKD', author_name.casefold())
|
||
if all(not unicodedata.category(char).startswith(cat) for cat in {'M', 'P', 'Z', 'C'}) or char == ' '
|
||
).casefold().upper()
|
||
author_name = re.sub("[^A-Z]", "", author_name)
|
||
side_identifier = "{:.3} {:.3}".format(author_name, title_normalized, )
|
||
if subtitle:
|
||
subtitle = re.sub(r'</span>', '', subtitle)
|
||
subtitle = re.sub(r'<span.*>', '', subtitle)
|
||
start = subtitle.split(' ')[0].replace('.', '')
|
||
start = re.sub("^R?", "", start)
|
||
|
||
if start.isnumeric():
|
||
side_identifier += " {:0>2}".format(start, )
|
||
|
||
# Normalize side identifier, in order to remove accents
|
||
side_identifier = ''.join(char for char in unicodedata.normalize('NFKD', side_identifier.casefold())
|
||
if all(not unicodedata.category(char).startswith(cat) for cat in {'M', 'P', 'Z', 'C'})
|
||
or char == ' ').casefold().upper()
|
||
|
||
return side_identifier
|
||
|
||
|
||
class MediaAdminForm(ModelForm):
|
||
def __init__(self, *args, **kwargs):
|
||
super().__init__(*args, **kwargs)
|
||
isbn_field = self.fields.get('isbn')
|
||
if isbn_field:
|
||
isbn_field.widget.template_name = "media/isbn_button.html"
|
||
isbn_field.widget.attrs.update({'autofocus': 'autofocus'})
|
||
side_identifier_field = self.fields.get('side_identifier')
|
||
if side_identifier_field and self.instance and self.instance.pk:
|
||
instance = self.instance
|
||
title, authors, subtitle = instance.title, instance.authors.all(), None
|
||
if hasattr(instance, "subtitle"):
|
||
subtitle = instance.subtitle
|
||
side_identifier_field.widget.attrs.update(
|
||
{'data-generated-side-identifier': generate_side_identifier(title, authors, subtitle)})
|
||
side_identifier_field.widget.template_name = "media/generate_side_identifier.html"
|
||
|
||
def download_data_isbndb(self, isbn):
|
||
api_url = "https://api2.isbndb.com/book/" + str(isbn) + "?Authorization=" + os.getenv("ISBNDB_KEY")
|
||
req = urllib.request.Request(api_url)
|
||
req.add_header("Authorization", os.getenv("ISBNDB_KEY"))
|
||
try:
|
||
with urllib.request.urlopen(req) as url:
|
||
data: dict = json.loads(url.read().decode())["book"]
|
||
except HTTPError:
|
||
return False
|
||
print(data)
|
||
data.setdefault("title", "")
|
||
data.setdefault("date_published", "1970-01-01")
|
||
data.setdefault("pages", 0)
|
||
data.setdefault("authors", [])
|
||
data.setdefault("image", "")
|
||
self.cleaned_data["title"] = data["title"]
|
||
self.cleaned_data["publish_date"] = data["date_published"][:10]
|
||
while len(self.cleaned_data["publish_date"]) == 4 or len(self.cleaned_data["publish_date"]) == 7:
|
||
self.cleaned_data["publish_date"] += "-01"
|
||
self.cleaned_data["number_of_pages"] = data["pages"]
|
||
self.cleaned_data["authors"] = \
|
||
list(Auteur.objects.get_or_create(name=author_name)[0] for author_name in data["authors"])
|
||
self.cleaned_data["external_url"] = data["image"]
|
||
return True
|
||
|
||
def download_data_bedeteque(self, isbn):
|
||
"""
|
||
Download data from bedeteque
|
||
:return True if success
|
||
"""
|
||
scraper = BedetequeScraper()
|
||
r = scraper.search_by_isbn(isbn)
|
||
if not r:
|
||
return False
|
||
# If results, then take the most accurate
|
||
data = scraper.scrap_bd_info(r[0])
|
||
self.cleaned_data.update(data)
|
||
return True
|
||
|
||
def download_data_google(self, isbn):
|
||
"""
|
||
Download data from google books
|
||
:return True if success
|
||
"""
|
||
api_url = "https://www.googleapis.com/books/v1/volumes?q=ISBN:{}"\
|
||
.format(isbn)
|
||
with urllib.request.urlopen(api_url) as url:
|
||
data = json.loads(url.read().decode())
|
||
|
||
if data and data['totalItems']:
|
||
fetched_item = None
|
||
for item in data['items']:
|
||
for identifiers in item["volumeInfo"]["industryIdentifiers"]:
|
||
if identifiers["identifier"] == isbn:
|
||
fetched_item = item
|
||
break
|
||
if fetched_item:
|
||
break
|
||
if not fetched_item:
|
||
return False
|
||
# Fill the data
|
||
self.parse_data_google(fetched_item)
|
||
return True
|
||
return False
|
||
|
||
def parse_data_google(self, data):
|
||
info = data['volumeInfo']
|
||
self.cleaned_data['external_url'] = info['canonicalVolumeLink']
|
||
if 'title' in info:
|
||
self.cleaned_data['title'] = info['title']
|
||
if 'subtitle' in data:
|
||
self.cleaned_data['subtitle'] = info['subtitle']
|
||
|
||
if 'pageCount' in info:
|
||
self.cleaned_data['number_of_pages'] = \
|
||
info['pageCount']
|
||
elif not self.cleaned_data['number_of_pages']:
|
||
self.cleaned_data['number_of_pages'] = 0
|
||
|
||
if 'publishedDate' in info:
|
||
if "-" not in info["publishedDate"]:
|
||
info["publishedDate"] += "-01-01"
|
||
elif len(info["publishedDate"]) == 7:
|
||
info["publishedDate"] += "-01"
|
||
self.cleaned_data['publish_date'] = info['publishedDate'][:10]
|
||
|
||
if 'authors' not in self.cleaned_data \
|
||
or not self.cleaned_data['authors']:
|
||
self.cleaned_data['authors'] = list()
|
||
|
||
if 'authors' in info:
|
||
for author in info['authors']:
|
||
author_obj = Auteur.objects.get_or_create(
|
||
name=author)[0]
|
||
self.cleaned_data['authors'].append(author_obj)
|
||
|
||
print(self.cleaned_data)
|
||
|
||
def download_data_openlibrary(self, isbn):
|
||
"""
|
||
Download data from openlibrary
|
||
:return True if success
|
||
"""
|
||
api_url = "https://openlibrary.org/api/books?bibkeys=ISBN:{}" \
|
||
"&format=json&jscmd=data".format(isbn)
|
||
with urllib.request.urlopen(api_url) as url:
|
||
data = json.loads(url.read().decode())
|
||
|
||
if data and data['ISBN:' + isbn]:
|
||
data = data['ISBN:' + isbn]
|
||
if 'url' in data:
|
||
# Fill the data
|
||
self.parse_data_openlibrary(data)
|
||
return True
|
||
return False
|
||
|
||
def parse_data_openlibrary(self, data):
|
||
self.cleaned_data['external_url'] = data['url']
|
||
if 'title' in data:
|
||
self.cleaned_data['title'] = data['title']
|
||
if 'subtitle' in data:
|
||
self.cleaned_data['subtitle'] = data['subtitle']
|
||
|
||
if 'number_of_pages' in data:
|
||
self.cleaned_data['number_of_pages'] = \
|
||
data['number_of_pages']
|
||
elif not self.cleaned_data['number_of_pages']:
|
||
self.cleaned_data['number_of_pages'] = 0
|
||
|
||
if 'publish_date' in data:
|
||
months = ['January', 'February', "March", "April", "Mai",
|
||
"June", "July", "August", "September",
|
||
"October", "November", "December"]
|
||
split = data['publish_date'].replace(',', '').split(' ')
|
||
if len(split) == 1:
|
||
self.cleaned_data['publish_date'] = split[0] + "-01-01"
|
||
else:
|
||
month_to_number = dict(
|
||
Jan="01",
|
||
Feb="02",
|
||
Mar="03",
|
||
Apr="04",
|
||
May="05",
|
||
Jun="06",
|
||
Jul="07",
|
||
Aug="08",
|
||
Sep="09",
|
||
Oct="10",
|
||
Nov="11",
|
||
Dec="12",
|
||
)
|
||
if split[0][:3] in month_to_number:
|
||
self.cleaned_data['publish_date']\
|
||
= split[2] + "-" \
|
||
+ month_to_number[split[0][:3]] + "-" + split[1]
|
||
else:
|
||
self.cleaned_data['publish_date'] = "{}-{:02d}-{:02d}" \
|
||
.format(split[2], months.index(split[0])
|
||
+ 1, int(split[1]), )
|
||
|
||
if 'authors' not in self.cleaned_data \
|
||
or not self.cleaned_data['authors']:
|
||
self.cleaned_data['authors'] = list()
|
||
|
||
if 'authors' in data:
|
||
for author in data['authors']:
|
||
author_obj = Auteur.objects.get_or_create(
|
||
name=author['name'])[0]
|
||
self.cleaned_data['authors'].append(author_obj)
|
||
|
||
def clean(self):
|
||
"""
|
||
If user fetch ISBN data, then download data before validating the form
|
||
"""
|
||
super().clean()
|
||
|
||
if "_isbn" in self.data\
|
||
or "_isbn_addanother" in self.data:
|
||
isbn = self.cleaned_data.get('isbn')
|
||
if "_isbn_addanother" in self.data:
|
||
self.data = self.data.copy()
|
||
self.data['_addanother'] = 42
|
||
self.request.POST = self.data
|
||
if isbn:
|
||
scrap_result = self.download_data_isbndb(isbn)
|
||
if not scrap_result:
|
||
# ISBN is present, try with bedeteque
|
||
scrap_result = self.download_data_bedeteque(isbn)
|
||
if not scrap_result:
|
||
# Try with Google
|
||
scrap_result = self.download_data_google(isbn)
|
||
if not scrap_result:
|
||
# Try with OpenLibrary
|
||
if not self.download_data_openlibrary(isbn):
|
||
self.add_error('isbn',
|
||
_("This ISBN is not found."))
|
||
return self.cleaned_data
|
||
|
||
if self.cleaned_data['title']:
|
||
self.cleaned_data['title'] = re.sub(
|
||
r'\(AUT\) ',
|
||
'',
|
||
self.cleaned_data['title']
|
||
)
|
||
|
||
if self.cleaned_data['authors']:
|
||
side_identifier = generate_side_identifier(
|
||
self.cleaned_data["title"],
|
||
self.cleaned_data["authors"],
|
||
self.cleaned_data["subtitle"],
|
||
)
|
||
|
||
self.cleaned_data['side_identifier'] = side_identifier
|
||
|
||
return self.cleaned_data
|
||
|
||
def _clean_fields(self):
|
||
for name, field in self.fields.items():
|
||
# value_from_datadict() gets the data from the data dictionaries.
|
||
# Each widget type knows how to retrieve its own data, because some
|
||
# widgets split data over several HTML fields.
|
||
if field.disabled:
|
||
value = self.get_initial_for_field(field, name)
|
||
else:
|
||
value = field.widget.value_from_datadict(
|
||
self.data, self.files, self.add_prefix(name))
|
||
from django.core.exceptions import ValidationError
|
||
try:
|
||
# We don't want to check a field when we enter an ISBN.
|
||
if "isbn" not in self.data \
|
||
or not self.cleaned_data.get('isbn'):
|
||
value = field.clean(value)
|
||
self.cleaned_data[name] = value
|
||
if hasattr(self, 'clean_%s' % name):
|
||
value = getattr(self, 'clean_%s' % name)()
|
||
self.cleaned_data[name] = value
|
||
except ValidationError as e:
|
||
self.add_error(name, e)
|
||
|
||
class Meta:
|
||
model = BD
|
||
fields = '__all__'
|