Add tqdm in GTFS import, add authorization code for CH-ALL GTF-RT import

requirements.txt

@@ -0,0 +1,7 @@
+Django>=5.0,<6.0
+django-cors-headers
+django-filter~=23.5
+djangorestframework~=3.14.0
+protobuf
+requests~=2.31.0
+tqdm

trainvel/gtfs/apps.py

@@ -1,6 +1,8 @@
 from django.apps import AppConfig
+from django.utils.translation import gettext_lazy as _
 
 
 class TrainvelGTFSConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
     name = "trainvel.gtfs"
+    verbose_name = _("Trainvel - GTFS")

trainvel/gtfs/locale/fr/LC_MESSAGES/django.po

@@ -2,7 +2,7 @@ msgid ""
 msgstr ""
 "Project-Id-Version: 1.0\n"
 "Report-Msgid-Bugs-To: \n"
-"POT-Creation-Date: 2024-05-09 19:34+0200\n"
+"POT-Creation-Date: 2024-05-09 22:04+0200\n"
 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 "Last-Translator: Emmy D'Anello <ynerant@emy.lu>\n"
 "Language-Team: LANGUAGE <LL@li.org>\n"
@@ -12,6 +12,10 @@ msgstr ""
 "Content-Transfer-Encoding: 8bit\n"
 "Plural-Forms: nplurals=2; plural=(n > 1);\n"
 
+#: trainvel/gtfs/apps.py:8
+msgid "Trainvel - GTFS"
+msgstr "Trainvel - GTFS"
+
 #: trainvel/gtfs/models.py:11
 msgid "Albania"
 msgstr "Albanie"

trainvel/gtfs/management/commands/update_sncf_gtfs.py

@@ -6,6 +6,7 @@ from zoneinfo import ZoneInfo
 
 import requests
 from django.core.management import BaseCommand
+from tqdm import tqdm
 
 from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, Stop, StopTime, Transfer, Trip, \
     PickupType
@@ -81,7 +82,7 @@ class Command(BaseCommand):
                     agencies.clear()
 
                 stops = []
-                for stop_dict in csv.DictReader(read_file("stops.txt")):
+                for stop_dict in csv.DictReader(tqdm(read_file("stops.txt"), desc="Stops")):
                     stop_dict: dict
                     stop_id = stop_dict['stop_id']
                     stop_id = f"{gtfs_code}-{stop_id}"
@@ -119,7 +120,7 @@ class Command(BaseCommand):
                     stops.clear()
 
                 routes = []
-                for route_dict in csv.DictReader(read_file("routes.txt")):
+                for route_dict in csv.DictReader(tqdm(read_file("routes.txt"), desc="Routes")):
                     route_dict: dict
                     route_id = route_dict['route_id']
                     route_id = f"{gtfs_code}-{route_id}"
@@ -154,10 +155,10 @@ class Command(BaseCommand):
                                               unique_fields=['id'])
                     routes.clear()
 
-                Calendar.objects.filter(gtfs_feed=gtfs_feed).delete()
+                # Calendar.objects.filter(gtfs_feed=gtfs_feed).delete()
                 calendars = {}
                 if "calendar.txt" in zipfile.namelist():
-                    for calendar_dict in csv.DictReader(read_file("calendar.txt")):
+                    for calendar_dict in csv.DictReader(tqdm(read_file("calendar.txt"), desc="Calendars")):
                         calendar_dict: dict
                         calendar = Calendar(
                             id=f"{gtfs_code}-{calendar_dict['service_id']}",
@@ -191,7 +192,7 @@ class Command(BaseCommand):
                         calendars.clear()
 
                 calendar_dates = []
-                for calendar_date_dict in csv.DictReader(read_file("calendar_dates.txt")):
+                for calendar_date_dict in csv.DictReader(tqdm(read_file("calendar_dates.txt"), desc="Calendar dates")):
                     calendar_date_dict: dict
                     calendar_date = CalendarDate(
                         id=f"{gtfs_code}-{calendar_date_dict['service_id']}-{calendar_date_dict['date']}",
@@ -238,7 +239,7 @@ class Command(BaseCommand):
                     calendar_dates.clear()
 
                 trips = []
-                for trip_dict in csv.DictReader(read_file("trips.txt")):
+                for trip_dict in csv.DictReader(tqdm(read_file("trips.txt"), desc="Trips")):
                     trip_dict: dict
                     trip_id = trip_dict['trip_id']
                     route_id = trip_dict['route_id']
@@ -277,7 +278,7 @@ class Command(BaseCommand):
                     trips.clear()
 
                 stop_times = []
-                for stop_time_dict in csv.DictReader(read_file("stop_times.txt")):
+                for stop_time_dict in csv.DictReader(tqdm(read_file("stop_times.txt"), desc="Stop times")):
                     stop_time_dict: dict
 
                     stop_id = stop_time_dict['stop_id']
@@ -336,7 +337,7 @@ class Command(BaseCommand):
 
                 if "transfers.txt" in zipfile.namelist():
                     transfers = []
-                    for transfer_dict in csv.DictReader(read_file("transfers.txt")):
+                    for transfer_dict in csv.DictReader(tqdm(read_file("transfers.txt"), desc="Transfers")):
                         transfer_dict: dict
                         from_stop_id = transfer_dict['from_stop_id']
                         to_stop_id = transfer_dict['to_stop_id']
@@ -367,7 +368,7 @@ class Command(BaseCommand):
                         transfers.clear()
 
                 if "feed_info.txt" in zipfile.namelist() and not dry_run:
-                    for feed_info_dict in csv.DictReader(read_file("feed_info.txt")):
+                    for feed_info_dict in csv.DictReader(tqdm(read_file("feed_info.txt"), desc="Feed info")):
                         feed_info_dict: dict
                         FeedInfo.objects.update_or_create(
                             publisher_name=feed_info_dict['feed_publisher_name'],

trainvel/gtfs/management/commands/update_sncf_gtfs_rt.py

@@ -4,6 +4,7 @@ from zoneinfo import ZoneInfo
 import requests
 from django.core.management import BaseCommand
 
+from trainvel import settings
 from trainvel.gtfs.gtfs_realtime_pb2 import FeedMessage, TripUpdate as GTFSTripUpdate
 from trainvel.gtfs.models import Agency, Calendar, CalendarDate, ExceptionType, GTFSFeed, PickupType, \
     Route, RouteType, Stop, StopScheduleRelationship, StopTime, StopTimeUpdate, \
@@ -26,8 +27,12 @@ class Command(BaseCommand):
             self.stdout.write(f"Updating GTFS-RT feed for {gtfs_feed}…")
 
             gtfs_code = gtfs_feed.code
+            headers = {}
+            if gtfs_code == "CH-ALL":
+                headers["Authorization"] = settings.OPENTRANSPORTDATA_SWISS_TOKEN
+            resp = requests.get(gtfs_feed.rt_feed_url, allow_redirects=True)
             feed_message = FeedMessage()
-            feed_message.ParseFromString(requests.get(gtfs_feed.rt_feed_url, allow_redirects=True).content)
+            feed_message.ParseFromString(resp.content)
 
             stop_times_updates = []
 

trainvel/settings.py

@@ -150,6 +150,8 @@ REST_FRAMEWORK = {
     'PAGE_SIZE': 20,
 }
 
+OPENTRANSPORTDATA_SWISS_TOKEN = "CHANGE ME"
+
 
 try:
     from .settings_local import *

trainvel/settings_local_example.py

@@ -18,3 +18,5 @@ DATABASES = {
         "PORT": "5432",
     }
 }
+
+OPENTRANSPORTDATA_SWISS_TOKEN = "CHANGE ME"