Support ODS and CSV formats to read notes from a spreadsheet

Signed-off-by: Emmy D'Anello <emmy.danello@animath.fr>

participation/forms.py

@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 
 import csv
+import math
 from io import StringIO
 import re
 from typing import Iterable
@@ -13,6 +14,7 @@ from django.contrib.auth.models import User
 from django.core.exceptions import ValidationError
 from django.core.validators import FileExtensionValidator
 from django.utils.translation import gettext_lazy as _
+import pandas
 from pypdf import PdfReader
 from registration.models import VolunteerRegistration
 
@@ -241,50 +243,50 @@ class AddJuryForm(forms.ModelForm):
 
 class UploadNotesForm(forms.Form):
     file = forms.FileField(
-        label=_("CSV file:"),
-        validators=[FileExtensionValidator(allowed_extensions=["csv"])],
+        label=_("Spreadsheet file:"),
+        validators=[FileExtensionValidator(allowed_extensions=["csv", "ods"])],
     )
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.fields['file'].widget.attrs['accept'] = 'text/csv'
+        self.fields['file'].widget.attrs['accept'] = 'text/csv,application/vnd.oasis.opendocument.spreadsheet'
 
     def clean(self):
         cleaned_data = super().clean()
 
         if 'file' in cleaned_data:
             file = cleaned_data['file']
-            with file:
-                try:
-                    data: bytes = file.read()
+            if file.name.endswith('.csv'):
+                with file:
                     try:
-                        content = data.decode()
-                    except UnicodeDecodeError:
-                        # This is not UTF-8, grrrr
-                        content = data.decode('latin1')
-                    for delimiter in [',', ';', '\t', '|']:
-                        if content.split('\n')[0].count(delimiter) > 1:
-                            break
-                    else:
-                        self.add_error('file',
-                                       _("Unable to detect the CSV delimiter. Please use a comma-separated file."))
-                        return cleaned_data
+                        data: bytes = file.read()
+                        try:
+                            content = data.decode()
+                        except UnicodeDecodeError:
+                            # This is not UTF-8, grrrr
+                            content = data.decode('latin1')
 
-                    csvfile = csv.reader(StringIO(content), delimiter=delimiter)
-                    self.process(csvfile, cleaned_data)
-                except UnicodeDecodeError:
-                    self.add_error('file', _("This file contains non-UTF-8 and non-ISO-8859-1 content. "
-                                             "Please send your sheet as a CSV file."))
+                        table = pandas.read_csv(StringIO(content), sep=None, header=None)
+                        self.process(table, cleaned_data)
+                    except UnicodeDecodeError:
+                        self.add_error('file', _("This file contains non-UTF-8 and non-ISO-8859-1 content. "
+                                                 "Please send your sheet as a CSV file."))
+            elif file.name.endswith('.ods'):
+                table = pandas.read_excel(file, header=None, engine='odf')
+                self.process(table, cleaned_data)
 
         return cleaned_data
 
-    def process(self, csvfile: Iterable[str], cleaned_data: dict):
+    def process(self, df: pandas.DataFrame, cleaned_data: dict):
         parsed_notes = {}
         valid_lengths = [2 + 6 * 3, 2 + 7 * 4, 2 + 6 * 5]  # Per pool sizes
         pool_size = 0
         line_length = 0
-        for line in csvfile:
-            line = [s.strip() for s in line if s]
+        for line in df.values.tolist():
+            # Remove NaN
+            line = [s for s in line if s == s]
+            # Strip cases
+            line = [str(s).strip() for s in line if str(s)]
             if line and line[0] == 'Problème':
                 pool_size = len(line) - 1
                 if pool_size < 3 or pool_size > 5:
@@ -297,12 +299,12 @@ class UploadNotesForm(forms.Form):
                 continue
 
             name = line[0]
-            if name.lower() in ["rôle", "juré", "moyenne", "coefficient", "sous-total", "équipe", "equipe"]:
+            if name.lower() in ["rôle", "juré⋅e", "juré?e", "moyenne", "coefficient", "sous-total", "équipe", "equipe"]:
                 continue
             notes = line[2:line_length]
             if not all(s.isnumeric() or s[0] == '-' and s[1:].isnumeric() for s in notes):
                 continue
-            notes = list(map(int, notes))
+            notes = list(map(lambda x: int(float(x)), notes))
 
             max_notes = pool_size * ([20, 20, 10, 10, 10, 10] + ([4] if pool_size == 4 else []))
             for n, max_n in zip(notes, max_notes):
@@ -312,7 +314,7 @@ class UploadNotesForm(forms.Form):
                                    + str(n) + " > " + str(max_n))
 
             # Search by volunteer id
-            jury = VolunteerRegistration.objects.filter(pk=line[1])
+            jury = VolunteerRegistration.objects.filter(pk=int(float(line[1])))
             if jury.count() != 1:
                 raise ValidationError({'file': _("The following user was not found:") + " " + name})
             jury = jury.get()

participation/templates/participation/pool_detail.html

@@ -133,7 +133,7 @@
                             <div class="btn btn-group">
                                 <button class="btn btn-sm btn-primary" data-bs-toggle="modal" data-bs-target="#uploadNotesModal">
                                     <i class="fas fa-upload"></i>
-                                    {% trans "Upload notes from a CSV file" %}
+                                    {% trans "Upload notes from a spreadsheet file" %}
                                 </button>
                                 <a class="btn btn-sm btn-info" href="{% url 'participation:pool_notes_template' pk=pool.pk %}">
                                     <i class="fas fa-download"></i>

participation/templates/participation/upload_notes.html

@@ -9,7 +9,6 @@
             <div class="alert alert-warning">
                 {% url 'participation:pool_jury' pk=pool.jury as jury_url %}
                 {% blocktrans trimmed with jury_url=jury_url %}
-                    Remember to export your spreadsheet as a CSV file before uploading it here.
                     Rows that are full of zeros are ignored.
                     Unknown juries are not considered.
                 {% endblocktrans %}