From b4f61308ab4b39e9d3fafdcc26da220c89553701 Mon Sep 17 00:00:00 2001 From: Emmy D'Anello Date: Thu, 9 May 2024 22:37:38 +0200 Subject: [PATCH] Add tqdm in GTFS import, add authorization code for CH-ALL GTF-RT import --- requirements.txt | 7 +++++++ trainvel/gtfs/apps.py | 2 ++ trainvel/gtfs/locale/fr/LC_MESSAGES/django.po | 6 +++++- .../management/commands/update_sncf_gtfs.py | 19 ++++++++++--------- .../commands/update_sncf_gtfs_rt.py | 7 ++++++- trainvel/settings.py | 2 ++ trainvel/settings_local_example.py | 2 ++ 7 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..36b3722 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Django>=5.0,<6.0 +django-cors-headers +django-filter~=23.5 +djangorestframework~=3.14.0 +protobuf +requests~=2.31.0 +tqdm \ No newline at end of file diff --git a/trainvel/gtfs/apps.py b/trainvel/gtfs/apps.py index bff9fdc..238230d 100644 --- a/trainvel/gtfs/apps.py +++ b/trainvel/gtfs/apps.py @@ -1,6 +1,8 @@ from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ class TrainvelGTFSConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "trainvel.gtfs" + verbose_name = _("Trainvel - GTFS") diff --git a/trainvel/gtfs/locale/fr/LC_MESSAGES/django.po b/trainvel/gtfs/locale/fr/LC_MESSAGES/django.po index edd247d..e1cf0a9 100644 --- a/trainvel/gtfs/locale/fr/LC_MESSAGES/django.po +++ b/trainvel/gtfs/locale/fr/LC_MESSAGES/django.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: 1.0\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-05-09 19:34+0200\n" +"POT-Creation-Date: 2024-05-09 22:04+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: Emmy D'Anello \n" "Language-Team: LANGUAGE \n" @@ -12,6 +12,10 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" +#: trainvel/gtfs/apps.py:8 +msgid "Trainvel - GTFS" +msgstr "Trainvel - GTFS" + #: trainvel/gtfs/models.py:11 msgid "Albania" msgstr "Albanie" diff --git a/trainvel/gtfs/management/commands/update_sncf_gtfs.py b/trainvel/gtfs/management/commands/update_sncf_gtfs.py index be2e913..3633f8a 100644 --- a/trainvel/gtfs/management/commands/update_sncf_gtfs.py +++ b/trainvel/gtfs/management/commands/update_sncf_gtfs.py @@ -6,6 +6,7 @@ from zoneinfo import ZoneInfo import requests from django.core.management import BaseCommand +from tqdm import tqdm from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, Stop, StopTime, Transfer, Trip, \ PickupType @@ -81,7 +82,7 @@ class Command(BaseCommand): agencies.clear() stops = [] - for stop_dict in csv.DictReader(read_file("stops.txt")): + for stop_dict in csv.DictReader(tqdm(read_file("stops.txt"), desc="Stops")): stop_dict: dict stop_id = stop_dict['stop_id'] stop_id = f"{gtfs_code}-{stop_id}" @@ -119,7 +120,7 @@ class Command(BaseCommand): stops.clear() routes = [] - for route_dict in csv.DictReader(read_file("routes.txt")): + for route_dict in csv.DictReader(tqdm(read_file("routes.txt"), desc="Routes")): route_dict: dict route_id = route_dict['route_id'] route_id = f"{gtfs_code}-{route_id}" @@ -154,10 +155,10 @@ class Command(BaseCommand): unique_fields=['id']) routes.clear() - Calendar.objects.filter(gtfs_feed=gtfs_feed).delete() + # Calendar.objects.filter(gtfs_feed=gtfs_feed).delete() calendars = {} if "calendar.txt" in zipfile.namelist(): - for calendar_dict in csv.DictReader(read_file("calendar.txt")): + for calendar_dict in csv.DictReader(tqdm(read_file("calendar.txt"), desc="Calendars")): calendar_dict: dict calendar = Calendar( id=f"{gtfs_code}-{calendar_dict['service_id']}", @@ -191,7 +192,7 @@ class Command(BaseCommand): calendars.clear() calendar_dates = [] - for calendar_date_dict in csv.DictReader(read_file("calendar_dates.txt")): + for calendar_date_dict in csv.DictReader(tqdm(read_file("calendar_dates.txt"), desc="Calendar dates")): calendar_date_dict: dict calendar_date = CalendarDate( id=f"{gtfs_code}-{calendar_date_dict['service_id']}-{calendar_date_dict['date']}", @@ -238,7 +239,7 @@ class Command(BaseCommand): calendar_dates.clear() trips = [] - for trip_dict in csv.DictReader(read_file("trips.txt")): + for trip_dict in csv.DictReader(tqdm(read_file("trips.txt"), desc="Trips")): trip_dict: dict trip_id = trip_dict['trip_id'] route_id = trip_dict['route_id'] @@ -277,7 +278,7 @@ class Command(BaseCommand): trips.clear() stop_times = [] - for stop_time_dict in csv.DictReader(read_file("stop_times.txt")): + for stop_time_dict in csv.DictReader(tqdm(read_file("stop_times.txt"), desc="Stop times")): stop_time_dict: dict stop_id = stop_time_dict['stop_id'] @@ -336,7 +337,7 @@ class Command(BaseCommand): if "transfers.txt" in zipfile.namelist(): transfers = [] - for transfer_dict in csv.DictReader(read_file("transfers.txt")): + for transfer_dict in csv.DictReader(tqdm(read_file("transfers.txt"), desc="Transfers")): transfer_dict: dict from_stop_id = transfer_dict['from_stop_id'] to_stop_id = transfer_dict['to_stop_id'] @@ -367,7 +368,7 @@ class Command(BaseCommand): transfers.clear() if "feed_info.txt" in zipfile.namelist() and not dry_run: - for feed_info_dict in csv.DictReader(read_file("feed_info.txt")): + for feed_info_dict in csv.DictReader(tqdm(read_file("feed_info.txt"), desc="Feed info")): feed_info_dict: dict FeedInfo.objects.update_or_create( publisher_name=feed_info_dict['feed_publisher_name'], diff --git a/trainvel/gtfs/management/commands/update_sncf_gtfs_rt.py b/trainvel/gtfs/management/commands/update_sncf_gtfs_rt.py index 0adaad6..3985dbe 100644 --- a/trainvel/gtfs/management/commands/update_sncf_gtfs_rt.py +++ b/trainvel/gtfs/management/commands/update_sncf_gtfs_rt.py @@ -4,6 +4,7 @@ from zoneinfo import ZoneInfo import requests from django.core.management import BaseCommand +from trainvel import settings from trainvel.gtfs.gtfs_realtime_pb2 import FeedMessage, TripUpdate as GTFSTripUpdate from trainvel.gtfs.models import Agency, Calendar, CalendarDate, ExceptionType, GTFSFeed, PickupType, \ Route, RouteType, Stop, StopScheduleRelationship, StopTime, StopTimeUpdate, \ @@ -26,8 +27,12 @@ class Command(BaseCommand): self.stdout.write(f"Updating GTFS-RT feed for {gtfs_feed}…") gtfs_code = gtfs_feed.code + headers = {} + if gtfs_code == "CH-ALL": + headers["Authorization"] = settings.OPENTRANSPORTDATA_SWISS_TOKEN + resp = requests.get(gtfs_feed.rt_feed_url, allow_redirects=True) feed_message = FeedMessage() - feed_message.ParseFromString(requests.get(gtfs_feed.rt_feed_url, allow_redirects=True).content) + feed_message.ParseFromString(resp.content) stop_times_updates = [] diff --git a/trainvel/settings.py b/trainvel/settings.py index 7dbc8bd..7e085b0 100644 --- a/trainvel/settings.py +++ b/trainvel/settings.py @@ -150,6 +150,8 @@ REST_FRAMEWORK = { 'PAGE_SIZE': 20, } +OPENTRANSPORTDATA_SWISS_TOKEN = "CHANGE ME" + try: from .settings_local import * diff --git a/trainvel/settings_local_example.py b/trainvel/settings_local_example.py index 5e12bf5..fd921ef 100644 --- a/trainvel/settings_local_example.py +++ b/trainvel/settings_local_example.py @@ -18,3 +18,5 @@ DATABASES = { "PORT": "5432", } } + +OPENTRANSPORTDATA_SWISS_TOKEN = "CHANGE ME"