Put content in a container

Signed-off-by: Emmy D'Anello <ynerant@emy.lu>

app.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
+from contextlib import nullcontext
 import csv
 from datetime import date, datetime, time, timedelta
 import os
 import json
 from pytz import timezone
 import requests
+from time import sleep
 
 import click
 from flask import Flask, render_template
@@ -18,13 +20,11 @@ from tqdm import tqdm
 
 import config
 
-
 app = Flask(__name__)
 
 cli = AppGroup('tgvmax', help="Manage the TGVMax dataset.")
 app.cli.add_command(cli)
 
-
 app.config |= config.FLASK_CONFIG
 
 db = SQLAlchemy(app)
@@ -61,53 +61,58 @@ class RouteQueue(db.Model):
 
 
 @cli.command("update-dataset")
-def update_dataset():
+@click.option('--verbose', '-v', is_flag=True, help="Display errors.")
+def update_dataset(verbose: bool = False):
     """
     Query the latest version of the SNCF OpenData dataset, as a CSV file.
     """
-    try:
+    resp = requests.get('https://ressources.data.sncf.com/explore/dataset/tgvmax/information/')
-        resp = requests.get('https://ressources.data.sncf.com/explore/dataset/tgvmax/information/')
+    content = resp.content.decode().split('<script type="application/ld+json">')[1].split('</script>')[0].strip()
-        content = resp.content.decode().split('<script type="application/ld+json">')[1].split('</script>')[0].strip()
+    content = content.replace('\r', '')
-        content = content.replace('\r', '')
+    content = content.replace('" \n', '" \\n')
-        content = content.replace('" \n', '" \\n')
+    content = content.replace('.\n', '.\\n')
-        content = content.replace('.\n', '.\\n')
+    content = content.replace('\n\n \nLa', '\\n\\n \\nLa')
-        content = content.replace('\n\n \nLa', '\\n\\n \\nLa')
+    content = content.replace('\n"', '\\n"')
-        content = content.replace('\n"', '\\n"')
 
-        info = json.loads(content)
+    info = json.loads(content)
-        modified_date = datetime.fromisoformat(info['dateModified'])
+    modified_date = datetime.fromisoformat(info['dateModified'])
 
-        utc = timezone('UTC')
+    utc = timezone('UTC')
-        last_modified = datetime.utcfromtimestamp(os.path.getmtime('tgvmax.csv')).replace(tzinfo=utc) if os.path.isfile('tgvmax.csv') else datetime(1, 1, 1, tzinfo=utc)
+    last_modified = datetime.utcfromtimestamp(os.path.getmtime('tgvmax.csv')).replace(tzinfo=utc) if os.path.isfile(
+        'tgvmax.csv') else datetime(1, 1, 1, tzinfo=utc)
 
-        if last_modified < modified_date:
+    if last_modified < modified_date:
+        if verbose:
             print("Updating tgvmax.csv…")
-            with requests.get(info['distribution'][0]['contentUrl'], stream=True) as resp:
+        with requests.get(info['distribution'][0]['contentUrl'], stream=True) as resp:
-                resp.raise_for_status()
+            resp.raise_for_status()
-                with open('tgvmax.csv', 'wb') as f:
+            with open('tgvmax.csv', 'wb') as f:
-                    with tqdm(unit='io', unit_scale=True) as t:
+                with tqdm(unit='io', unit_scale=True) if verbose else nullcontext() as t:
-                        for chunk in resp.iter_content(chunk_size=512 * 1024):
+                    for chunk in resp.iter_content(chunk_size=512 * 1024):
-                            if chunk:
+                        if chunk:
-                                f.write(chunk)
+                            f.write(chunk)
+                            if verbose:
                                 t.update(len(chunk))
-            os.utime('tgvmax.csv', (modified_date.timestamp(), modified_date.timestamp()))
+        os.utime('tgvmax.csv', (modified_date.timestamp(), modified_date.timestamp()))
+        if verbose:
             print("Done")
-
+    else:
-        print("Last modification:", modified_date)
+        if verbose:
-    except Exception as e:
+            print("Last modification:", modified_date)
-        print("An error occured while updating tgvmax.csv")
+        exit(2)
-        print(e)
 
 
 @cli.command("parse-csv")
 @click.option('-F', '--flush', type=bool, is_flag=True, help="Flush the database before filling it.")
-def parse_trains(flush: bool = False):
+@click.option('--verbose', '-v', is_flag=True, help="Display errors.")
+def parse_trains(flush: bool = False, verbose: bool = False):
     """
     Parse the CSV file and store it to the database.
     """
 
     if flush:
-        print("Flush database…")
+        if verbose:
+            print("Flush database…")
         db.session.query(Train).delete()
 
     last_modification = datetime.utcfromtimestamp(os.path.getmtime('tgvmax.csv')).replace(tzinfo=timezone('UTC'))
@@ -115,8 +120,11 @@ def parse_trains(flush: bool = False):
     with open('tgvmax.csv') as f:
         first_line = True
         already_seen = set()
-        for line in tqdm(csv.reader(f, delimiter=';')):
+        already_updated = set(x[0] for x in db.session.query(Train).filter(Train.last_modification > last_modification)\
+                .values(Train.id))
+        for line in (tqdm if verbose else lambda x: x)(csv.reader(f, delimiter=';')):
             if first_line:
+                # Skip first line
                 first_line = False
                 continue
 
@@ -129,6 +137,10 @@ def parse_trains(flush: bool = False):
                     print("Duplicate:", train_id)
                 continue
 
+            if train_id in already_updated:
+                # Already updated by the simulator
+                continue
+
             train = Train(
                 id=train_id,
                 day=date.fromisoformat(line[0]),
@@ -156,59 +168,69 @@ def parse_trains(flush: bool = False):
     db.session.commit()
 
 
-def find_routes(day, origin, destination):
+def find_routes(day: date | datetime, origin: str, destination: str | None,
-    trains = db.session.query(Train).filter_by(day=day, tgvmax=True).all()
+                verbose: bool = False, min_dep: time = time(0, 0),
+                explored: dict | None = None):
+    if isinstance(day, datetime):
+        day = day.date()
+
+    if explored is None:
+        explored = {}
+
+    if origin not in explored:
+        explored[origin] = (min_dep, None)
+        valid_routes = []
+        max_dep = time(23, 59)
+    else:
+        max_dep, valid_routes = explored[origin]
+        if max_dep < min_dep:
+            # Already parsed these trains
+            return {destination: valid_routes}
+        explored[origin] = min_dep, None
+
+    trains = db.session.query(Train).filter_by(day=day, tgvmax=True, orig=origin)\
+            .filter(Train.dep >= min_dep, Train.dep < max_dep).all()
+    if not trains:
+        # No train in the requested interval
+        explored[origin] = (min_dep, valid_routes)
+        return {destination: valid_routes}
 
     trains.sort(key=lambda train: train.dep)
+    db.session.commit()
 
-    explore = []
+    for train in (t := tqdm(trains, desc=origin) if verbose else trains):
-    per_arr_explore = {}
+        if train.dest == destination:
-    valid_routes = []
+            # We hope that we have a direct train
-
+            valid_routes.append([train])
-    for train in tqdm(trains):
+        else:
-        if train.orig == origin:
+            if train.dest in explored and explored[train.dest][1] is None:
-            it = [train]
+                # This is a loop
-            if train.dest == destination:
-                # We hope that we have a direct train
-                valid_routes.append(it)
-            else:
-                explore.append(it)
-                per_arr_explore.setdefault(train.dest, [])
-                per_arr_explore[train.dest].append(it)
-            continue
-
-        for it in list(per_arr_explore.get(train.orig, [])):
-            if any(train.dest == tr.dest or train.dest == origin for tr in it):
-                # Avoid loops
                 continue
+            elif train.arr < min_dep:
+                # The train is not direct and arrives on the next day, we avoid that
+                continue
+            find_routes(day, train.dest, destination, verbose, train.arr, explored)
 
-            last_train = it[-1]
+            # Filter unusuable trains
+            valid_routes += [[train] + it for it in explored[train.dest][1] if it[0].dep >= train.arr]
 
-            if last_train.arr <= train.dep:
+    explored[origin] = (min_dep, valid_routes)
-                new_it = it + [train]
-                if train.dest == destination:
-                    # Goal is achieved
-                    valid_routes.append(new_it)
-                else:
-                    explore.append(new_it)
-                    per_arr_explore.setdefault(train.dest, [])
-                    per_arr_explore[train.dest].append(new_it)
 
-    return valid_routes
+    # Send queued trains to the database
+    db.session.commit()
+
+    return {destination: valid_routes} if destination else {}
 
 
-def print_route(route: list[Train]):
+# Don't use the decorator to keep the function callable
-    s = f"{route[0].orig} "
+cli.command('find-routes')(click.argument('day', type=click.DateTime(formats=['%Y-%m-%d']))
-    for tr in route:
+                           (click.argument('origin', type=str)
-        s += f"({tr.dep}) --> ({tr.arr}) {tr.dest}, "
+                            (click.argument('destination', type=str, default=None)
-    print(s[:-2])
+                             (click.option('--verbose', '-v', type=bool, is_flag=True, help="Display errors.")
+                              (find_routes)))))
 
 
-@cli.command('queue-route')
+def queue_route(day: date | datetime, origin: str, destination: str, verbose: bool = False, autocommit: bool = True):
-@click.argument('day', type=click.DateTime(formats=['%Y-%m-%d']))
-@click.argument('origin', type=str)
-@click.argument('destination', type=str)
-def queue_route(day: date | datetime, origin: str, destination: str):
     """
     Fetch the TGVMax simulator to refresh data.
 
@@ -221,44 +243,89 @@ def queue_route(day: date | datetime, origin: str, destination: str):
     if isinstance(day, datetime):
         day = day.date()
 
-    query = db.session.query(RouteQueue).filter_by(day=day, origin=origin, destination=destination, response_time=None)
+    query = db.session.query(RouteQueue).filter_by(day=day, origin=origin, destination=destination)\
+            .filter((RouteQueue.response_time == None) | (RouteQueue.expiration_time >= datetime.now(timezone('UTC'))))
     if query.count():
-        print("Already queued")
-        return
-
-    query = db.session.query(RouteQueue).filter(RouteQueue.day == day,
-                                                RouteQueue.origin == origin,
-                                                RouteQueue.destination == destination,
-                                                RouteQueue.expiration_time >= datetime.now(timezone('UTC')))
-    if query.count():
-        print("Using recent value")
         return
 
     db.session.add(RouteQueue(day=day, origin=origin, destination=destination))
-    db.session.commit()
+    if autocommit:
+        db.session.commit()
+
+
+# Don't use the decorator to keep the function callable
+cli.command('queue-route')(click.argument('day', type=click.DateTime(formats=['%Y-%m-%d']))
+                           (click.argument('origin', type=str)
+                            (click.argument('destination', type=str)
+                             (click.option('--verbose', '-v', type=bool, is_flag=True, help="Display errors.")
+                              (queue_route)))))
+
+
+def queue_routes(day: date | datetime, origin: str | None = None,
+                 destination: str | None = None, verbose: bool = False,
+                 autocommit: bool = True):
+    if isinstance(day, datetime):
+        day = day.date()
+
+    valid_routes = set(db.session.query(RouteQueue).filter_by(day=day)\
+            .filter((RouteQueue.response_time == None) | (RouteQueue.expiration_time >= datetime.now(timezone('UTC'))))\
+            .values(RouteQueue.origin, RouteQueue.destination))
+
+    query = db.session.query(Train).filter((Train.day == day))
+    if origin:
+        query = query.filter((Train.orig_iata == origin) | (Train.orig == origin))
+    if destination:
+        query = query.filter((Train.dest_iata == destination) | (Train.dest == destination))
+    query = query.all()
+    for train in (t := tqdm(query) if verbose else query):
+        if verbose:
+            t.set_description(f"{day}: {train.orig} --> {train.dest}")
+        if (train.orig_iata, train.dest_iata) not in valid_routes:
+            queue_route(day, train.orig_iata, train.dest_iata, verbose, autocommit)
+            valid_routes.add((train.orig_iata, train.dest_iata))
+
+
+# Same as above
+cli.command('queue-routes')(click.argument('day', type=click.DateTime(formats=['%Y-%m-%d']))
+                            (click.option('--origin', '-o', default=None)
+                             (click.option('--destination', '-d', default=None)
+                              (click.option('--verbose', '-v', type=bool, is_flag=True, help="Display errors.")
+                               (queue_routes)))))
 
 
 @cli.command('process-queue', help="Process the waiting list to refresh from the simulator.")
-@click.argument('number', default=5, type=int)
+@click.argument('number', default=30, type=int)
-def process_queue(number: int):
+@click.option('--verbose', '-v', type=bool, is_flag=True, help="Display errors.")
-    queue = db.session.query(RouteQueue).filter_by(response_time=None).order_by(RouteQueue.queue_time)
+def process_queue(number: int, verbose: bool = False):
+    queue = db.session.query(RouteQueue).filter_by(response_time=None).order_by(RouteQueue.queue_time).all()
     if number > 0:
         queue = queue[:number]
 
     URL = "https://www.maxjeune-tgvinoui.sncf/api/public/refdata/search-freeplaces-proposals"
 
-    for req in queue:
+    if verbose:
+        query = db.session.query(Train).with_entities(Train.orig_iata, Train.orig).distinct()
+        iata_to_names = {k: v for (k, v) in query.all()}
+
+    for i, req in enumerate(t := tqdm(queue) if verbose else queue):
         req: RouteQueue
-        resp = requests.post(URL, json={
+        if verbose:
-            'departureDateTime': req.day.isoformat(),
+            t.set_description(f"{req.day:%d/%m/%Y}: {iata_to_names[req.origin]} --> {iata_to_names[req.destination]}")
-            'origin': req.origin,
+
-            'destination': req.destination,
+        resp = None
-        })
+        while resp is None or resp.status_code == 429:
+            resp = requests.post(URL, json={
+                'departureDateTime': req.day.isoformat(),
+                'origin': req.origin,
+                'destination': req.destination,
+            })
+            if resp.status_code == 429:
+                sleep(1)
 
         if resp.status_code == 404:
             # No travel found
             req.response_time = datetime.now()
-            req.expiration_time = datetime.now() + timedelta(hours=1)
+            req.expiration_time = datetime.now() + timedelta(hours=3)
             db.session.add(req)
             continue
 
@@ -270,18 +337,27 @@ def process_queue(number: int):
         req.expiration_time += timedelta(hours=3)  # By default 5 minutes, extend it to 3 hours to be safe
         db.session.add(req)
 
-        db.session.query(Train).filter_by(day=req.day, orig_iata=req.origin, dest_iata=req.destination)\
+        db.session.query(Train).filter_by(day=req.day, orig_iata=req.origin, dest_iata=req.destination) \
             .update(dict(tgvmax=False, remaining_seats=-1))
 
         for proposal in data['proposals']:
             train = db.session.query(Train).filter_by(day=req.day, number=int(proposal['trainNumber']),
-                                                      orig_iata=req.origin, dest_iata=req.destination).first()
+                                                      orig_iata=proposal['origin']['rrCode'],
+                                                      dest_iata=proposal['destination']['rrCode']).first()
+            if train is None:
+                # In a city with multiple stations
+                print("ERROR")
+                print(proposal)
+                continue
             train.tgvmax = True
             train.remaining_seats = proposal['freePlaces']
             train.last_modification = req.response_time
             train.expiration_time = req.expiration_time
             db.session.add(train)
 
+        if i % 50 == 0:
+            db.session.commit()
+
     db.session.commit()
 
 
@@ -304,20 +380,29 @@ def iata_codes():
 
 
 @app.get('/api/routes/<day>/<origin>/<destination>/')
-def get_routes(day: date, origin: str, destination: str):
+def get_routes(day: date | str, origin: str, destination: str):
+    if isinstance(day, str):
+        day = date.fromisoformat(day)
+
+    if destination == 'undefined':
+        destination = None
+
     routes = find_routes(day, origin, destination)
-    return [
+
-        [{
+    return {
-            'origin': tr.orig,
+            city: [
-            'origin_iata': tr.orig_iata,
+                [{
-            'destination': tr.dest,
+                    'origin': tr.orig,
-            'destination_iata': tr.dest_iata,
+                    'origin_iata': tr.orig_iata,
-            'departure': tr.dep.isoformat(),
+                    'destination': tr.dest,
-            'arrival': tr.arr.isoformat(),
+                    'destination_iata': tr.dest_iata,
-            'number': tr.number,
+                    'departure': tr.dep.isoformat(),
-            'free_seats': tr.remaining_seats,
+                    'arrival': tr.arr.isoformat(),
-          } for tr in route] for route in routes
+                    'number': tr.number,
-    ]
+                    'free_seats': tr.remaining_seats,
+                } for tr in route] for route in city_routes
+            ] for city, city_routes in routes.items()
+        }
 
 
 if __name__ == '__main__':

static/bootstrap5

@@ -0,0 +1 @@
+/usr/share/javascript/bootstrap5/

templates/index.html

@@ -2,53 +2,50 @@
 <html lang="fr">
 <head>
     <title>Calculateur TGVMax</title>
-    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha1/dist/css/bootstrap.min.css"
+    <link href="/static/bootstrap5/css/bootstrap.min.css" rel="stylesheet">
-          rel="stylesheet"
-          integrity="sha384-GLhlTQ8iRABdZLl6O3oVMWSktQOp6b7In1Zl3/Jr59b6EGGoI1aFkw7cmDA6j6gD"
-          crossorigin="anonymous">
 </head>
 <body>
-    <form id="form" action="#">
+    <main class="container">
+        <form id="form" action="#">
         <datalist id="iataCodes">
             <option value="Chargement…">
         </datalist>
         <div class="mb-3">
             <label for="origin" class="form-label">Origine :</label>
             <input type="text" class="form-control" list="iataCodes" id="origin"
-                   placeholder="Origine…" aria-describedby="originHelp">
+               placeholder="Origine…" aria-describedby="originHelp">
             <input type="hidden" class="form-control" id="originIata">
             <div id="originHelp" class="form-text">Le point de départ de votre trajet.</div>
         </div>
         <div class="mb-3">
             <label for="destination" class="form-label">Destination :</label>
             <input type="text" class="form-control" list="iataCodes" id="destination"
-                   placeholder="Destination…" aria-describedby="destinationHelp">
+               placeholder="Destination…" aria-describedby="destinationHelp">
             <input type="hidden" class="form-control" id="destinationIata">
             <div id="destinationHelp" class="form-text">Le point d'arrivée de votre trajet.</div>
         </div>
         <div class="mb-3">
             <label for="day" class="form-label">Jour de départ :</label>
             <input type="date" class="form-control" id="day" aria-describedby="dayHelp"
-                   min="{{ today }}" max="{{ max_day }}" value="{{ today }}">
+               min="{{ today }}" max="{{ max_day }}" value="{{ today }}">
             <div id="dayHelp" class="form-text">Le jour de votre départ.</div>
         </div>
         <div class="mb-3">
             <input type="submit" class="form-control btn btn-primary" value="Rechercher…">
         </div>
-    </form>
+        </form>
-    <div id="result"></div>
+        <div id="result"></div>
+    </main>
 </body>
 
-<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0-alpha1/dist/js/bootstrap.bundle.min.js"
+<script src="/static/bootstrap5/js/bootstrap.bundle.min.js"></script>
-        integrity="sha384-w76AqPfDkMBDXo30jS1Sgez6pr3x5MlQ1ZAGC+nuZB+EYdgRZgiwxhTBTkF7CXvN"
-        crossorigin="anonymous"></script>
 
 <script>
     fetch('/api/iata-codes/').then(res => res.json()).then(out => {
         let datalist = document.getElementById('iataCodes')
         datalist.innerHTML = ''
-        for (let iata in out.iata2name) {
+        for (let name in out.name2iata) {
-            let name = out.iata2name[iata]
+            let iata = out.name2iata[name]
             let elem = document.createElement('option')
             elem.value = name
             elem.setAttribute('data-iata', iata)
@@ -77,27 +74,71 @@
     let day_elem = document.getElementById('day')
     let result_elem = document.getElementById('result')
     document.getElementById('form').addEventListener('submit', () => {
-        result_elem.innerHTML = 'Chargement…'
+        result_elem.innerHTML = '<div class="spinner-border" role="status"><span class="visually-hidden">Chargement…</span></div>'
-        fetch('/api/routes/' + day_elem.value + '/' + origin_elem.value + '/' + destination_elem.value + '/')
+        fetch('/api/routes/' + day_elem.value + '/' + origin_elem.value + '/' + (destination_elem.value || 'undefined') + '/')
             .then(resp => resp.json())
             .then(routes => {
-                console.log(routes)
                 result_elem.innerHTML = ''
-                let routes_elem = document.createElement('ul')
-                result_elem.appendChild(routes_elem)
-                for (let route in routes) {
-                    route = routes[route]
-                    console.log(route)
-                    let route_elem = document.createElement('li')
-                    routes_elem.appendChild(route_elem)
 
-                    let text = route[0].origin
+                let cities_number = Object.keys(routes).length
-                    for (let train in route) {
+
-                        train = route[train]
+                let accordion = document.createElement('div')
-                        console.log(train)
+                accordion.classList.add('accordion')
-                        text += " (" + train.departure + ") --> (" + train.arrival + ") " + train.destination + ", "
+                accordion.setAttribute('id', 'result-accordion')
+                result_elem.appendChild(accordion)
+
+                for (let city in routes) {
+                    let city_routes = routes[city]
+
+                    let city_id = city.replaceAll(' ', '_')
+
+                    let city_elem = document.createElement('div')
+                    city_elem.classList.add('accordion-item')
+                    accordion.appendChild(city_elem)
+
+                    let city_header = document.createElement('h2')
+                    city_header.classList.add('accordion-header')
+                    city_header.setAttribute('id', 'collapse-header-' + city_id)
+                    city_elem.appendChild(city_header)
+
+                    let city_name_button = document.createElement('button')
+                    city_name_button.classList.add('accordion-button')
+                    if (cities_number > 1)
+                        city_name_button.classList.add('collapsed')
+                    city_name_button.setAttribute('type', 'button')
+                    city_name_button.setAttribute('data-bs-toggle', 'collapse')
+                    city_name_button.setAttribute('data-bs-target', '#collapse-' + city_id)
+                    city_name_button.setAttribute('aria-expanded', cities_number > 1 ? 'false' : 'true')
+                    city_name_button.setAttribute('aria-controls', 'collapse-' + city_id)
+                    city_name_button.textContent = city
+                    city_header.appendChild(city_name_button)
+
+                    let collapse = document.createElement('div')
+                    collapse.setAttribute('id', 'collapse-' + city_id)
+                    collapse.classList.add('accordion-collapse')
+                    if (cities_number > 1)
+                        collapse.classList.add('collapse')
+                    collapse.setAttribute('aria-labelled-by', 'collapse-header-' + city_id)
+                    collapse.setAttribute('data-bs-parent', '#result-accordion')
+                    city_elem.appendChild(collapse)
+
+                    let collapse_body = document.createElement('div')
+                    collapse_body.classList.add('accordion-body')
+                    collapse.appendChild(collapse_body)
+
+                    let routes_elem = document.createElement('ul')
+                    collapse_body.appendChild(routes_elem)
+
+                    for (let route of city_routes) {
+                        let route_elem = document.createElement('li')
+                        routes_elem.appendChild(route_elem)
+
+                        let text = route[0].origin
+                        for (let train of route) {
+                            text += " (" + train.departure + ") --> (" + train.arrival + ") " + train.destination + " "
+                        }
+                        route_elem.textContent = text
                     }
-                    route_elem.textContent = text
                 }
             })
     })