From 16520c36644d166e4d8028a43df1d8836fe77016 Mon Sep 17 00:00:00 2001 From: Emmy D'Anello Date: Sat, 10 Feb 2024 17:33:36 +0100 Subject: [PATCH] =?UTF-8?q?Prepare=20code=20for=20Eurostar/Trenitalia=20Fr?= =?UTF-8?q?ance/RENFE/=C3=96BB=20data=20input?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sncf-station/public/bus.svg | 5 + sncf-station/public/eurostar.svg | 15 ++ sncf-station/public/eurostar_mini.svg | 7 + sncf-station/public/frecciarossa.svg | 18 ++ sncf-station/public/ice.svg | 9 + sncf-station/public/nightjet.svg | 18 ++ sncf-station/public/ouigo.svg | 52 +++++ sncf-station/public/renfe.svg | 23 +++ sncf-station/public/ter.svg | 6 + sncf-station/public/tgv_inoui.svg | 2 + sncf-station/public/trenitalia.svg | 17 ++ sncf-station/src/AutocompleteStop.jsx | 23 ++- sncf-station/src/Station.js | 3 +- sncf-station/src/TrainsTable.js | 118 ++++++++--- sncf/api/views.py | 19 +- sncfgtfs/admin.py | 6 +- .../management/commands/update_sncf_gtfs.py | 185 ++++++++++++------ sncfgtfs/migrations/0001_initial.py | 34 +++- sncfgtfs/models.py | 11 +- 19 files changed, 463 insertions(+), 108 deletions(-) create mode 100644 sncf-station/public/bus.svg create mode 100644 sncf-station/public/eurostar.svg create mode 100644 sncf-station/public/eurostar_mini.svg create mode 100644 sncf-station/public/frecciarossa.svg create mode 100644 sncf-station/public/ice.svg create mode 100644 sncf-station/public/nightjet.svg create mode 100644 sncf-station/public/ouigo.svg create mode 100644 sncf-station/public/renfe.svg create mode 100644 sncf-station/public/ter.svg create mode 100644 sncf-station/public/tgv_inoui.svg create mode 100644 sncf-station/public/trenitalia.svg diff --git a/sncf-station/public/bus.svg b/sncf-station/public/bus.svg new file mode 100644 index 0000000..ccde2e1 --- /dev/null +++ b/sncf-station/public/bus.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/sncf-station/public/eurostar.svg b/sncf-station/public/eurostar.svg new file mode 100644 index 0000000..39c219c --- /dev/null +++ b/sncf-station/public/eurostar.svg @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/eurostar_mini.svg b/sncf-station/public/eurostar_mini.svg new file mode 100644 index 0000000..6624ca6 --- /dev/null +++ b/sncf-station/public/eurostar_mini.svg @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/frecciarossa.svg b/sncf-station/public/frecciarossa.svg new file mode 100644 index 0000000..a7de6d2 --- /dev/null +++ b/sncf-station/public/frecciarossa.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/ice.svg b/sncf-station/public/ice.svg new file mode 100644 index 0000000..1cc604b --- /dev/null +++ b/sncf-station/public/ice.svg @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/nightjet.svg b/sncf-station/public/nightjet.svg new file mode 100644 index 0000000..8bd3a0b --- /dev/null +++ b/sncf-station/public/nightjet.svg @@ -0,0 +1,18 @@ + + + + + + \ No newline at end of file diff --git a/sncf-station/public/ouigo.svg b/sncf-station/public/ouigo.svg new file mode 100644 index 0000000..1efd32a --- /dev/null +++ b/sncf-station/public/ouigo.svg @@ -0,0 +1,52 @@ + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/renfe.svg b/sncf-station/public/renfe.svg new file mode 100644 index 0000000..df95b54 --- /dev/null +++ b/sncf-station/public/renfe.svg @@ -0,0 +1,23 @@ + + + +Renfe + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/public/ter.svg b/sncf-station/public/ter.svg new file mode 100644 index 0000000..3b3c0cd --- /dev/null +++ b/sncf-station/public/ter.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/sncf-station/public/tgv_inoui.svg b/sncf-station/public/tgv_inoui.svg new file mode 100644 index 0000000..5c60852 --- /dev/null +++ b/sncf-station/public/tgv_inoui.svg @@ -0,0 +1,2 @@ + +image/svg+xml \ No newline at end of file diff --git a/sncf-station/public/trenitalia.svg b/sncf-station/public/trenitalia.svg new file mode 100644 index 0000000..ba8641c --- /dev/null +++ b/sncf-station/public/trenitalia.svg @@ -0,0 +1,17 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/sncf-station/src/AutocompleteStop.jsx b/sncf-station/src/AutocompleteStop.jsx index 5b8fed9..675a305 100644 --- a/sncf-station/src/AutocompleteStop.jsx +++ b/sncf-station/src/AutocompleteStop.jsx @@ -32,11 +32,32 @@ function AutocompleteStop(params) { filterOptions={(x) => x} getOptionKey={option => option.id} getOptionLabel={option => option.name} - groupBy={option => option.id.startsWith("IDFM") ? "Transilien" : "TER/TGV/Intercités"} + groupBy={option => getOptionGroup(option)} isOptionEqualToValue={(option, value) => option.id === value.id} renderInput={(params) => } {...params} /> } +function getOptionGroup(option) { + switch (option.transport_type) { + case "TGV": + case "IC": + case "TER": + return "TGV/TER/Intercités" + case "TN": + return "Transilien" + case "ES": + return "Eurostar" + case "TI": + return "Trenitalia France" + case "RENFE": + return "RENFE" + case "OBB": + return "ÖBB" + default: + return option.transport_type + } +} + export default AutocompleteStop; diff --git a/sncf-station/src/Station.js b/sncf-station/src/Station.js index 080af9f..a5049e3 100644 --- a/sncf-station/src/Station.js +++ b/sncf-station/src/Station.js @@ -11,7 +11,8 @@ function DateTimeSelector({stop, date, time}) { const navigate = useNavigate() function onStationSelected(event, stop) { - navigate(`/station/${stop.id}/`) + if (stop !== null) + navigate(`/station/${stop.id}/`) } return <> diff --git a/sncf-station/src/TrainsTable.js b/sncf-station/src/TrainsTable.js index de1cd05..6c39561 100644 --- a/sncf-station/src/TrainsTable.js +++ b/sncf-station/src/TrainsTable.js @@ -107,7 +107,11 @@ function TrainRow({train, tableType, date, time}) { enabled: !!trip.route, }) const route = routeQuery.data ?? {} - const trainType = getTrainType(train, route) + const trainType = getTrainType(train, trip, route) + const backgroundColor = getBackgroundColor(train, trip, route) + console.log(backgroundColor) + const textColor = getTextColor(train, trip, route) + const trainTypeDisplay = getTrainTypeDisplay(trainType) const stopTimesQuery = useQuery({ queryKey: ['stop_times', trip.id], @@ -151,7 +155,6 @@ function TrainRow({train, tableType, date, time}) { const delay = tableType === "departures" ? realtimeData.departure_delay : realtimeData.arrival_delay const prettyDelay = delay && scheduleRelationship !== 3 ? getPrettyDelay(delay) : "" const [prettyScheduleRelationship, scheduleRelationshipColor] = getPrettyScheduleRelationship(scheduleRelationship) - console.log(realtimeTripData) let stopsFilter if (scheduleRelationship === 3) @@ -174,9 +177,9 @@ function TrainRow({train, tableType, date, time}) { height="4em" borderRadius="15%" fontWeight="bold" - backgroundColor={`#${getBackgroundColor(train, route)}`} - color={`#${getTextColor(train, route)}`}> - {trainType} + backgroundColor={backgroundColor} + color={textColor}> + {trainTypeDisplay} @@ -211,46 +214,99 @@ function TrainRow({train, tableType, date, time}) { } -function getTrainType(train, route) { - if (train.id.startsWith("IDFM")) - return route.short_name - else { - let trainType = train.stop.split("StopPoint:OCE")[1].split("-")[0] - if (trainType === "Train TER") - trainType = "TER" - else if (trainType === "INTERCITES") - trainType = "INTER-CITÉS" - else if (trainType === "INTERCITES de nuit") - trainType = "INTER-CITÉS de nuit" - return trainType +function getTrainType(train, trip, route) { + switch (route.transport_type) { + case "TGV": + case "TER": + case "IC": + let trainType = train.stop.split("StopPoint:OCE")[1].split("-")[0] + switch (trainType) { + case "Train TER": + return "TER" + case "INTERCITES": + return "INTER-CITÉS" + case "INTERCITES de nuit": + return "INTER-CITÉS de nuit" + default: + return trainType + } + case "TN": + return route.short_name + case "ES": + return "Eurostar" + case "TI": + return "Trenitalia" + case "RENFE": + return "RENFE" + case "OBB": + if (trip.short_name.startsWith("NJ")) + return "NJ" + return "ÖBB" + default: + return "" } } -function getBackgroundColor(train, route) { - if (route.color) - return route.color - else if (getTrainType(train, route) === "OUIGO") - return "E60075" - return "FFFFFF" +function getTrainTypeDisplay(trainType) { + switch (trainType) { + case "TGV INOUI": + return TGV INOUI + case "OUIGO": + return OUIGO + case "TER": + return TER + case "Car TER": + return
Car +
+ TER
+ case "ICE": + return ICE + case "Eurostar": + return Eurostar + case "Trenitalia": + return Frecciarossa + case "RENFE": + return RENFE + case "NJ": + return NightJet + default: + return trainType + } } -function getTextColor(train, route) { +function getBackgroundColor(train, trip, route) { + let trainType = getTrainType(train, trip, route) + switch (trainType) { + case "OUIGO": + return "#0096CA" + case "Eurostar": + return "#00286A" + case "NJ": + return "#272759" + default: + if (route.color) + return `#${route.color}` + return "#FFFFFF" + } +} + +function getTextColor(train, trip, route) { if (route.text_color) - return route.text_color + return `#${route.text_color}` else { - let trainType = getTrainType(train, route) + let trainType = getTrainType(train, trip, route) switch (trainType) { case "OUIGO": - return "FFFFFF" + return "#FFFFFF" case "TGV INOUI": - return "9B2743" + return "#9B2743" case "ICE": - return "B4B4B4" + return "#B4B4B4" case "INTER-CITÉS": case "INTER-CITÉS de nuit": - return "404042" + return "#404042" default: - return "000000" + return "#000000" } } } diff --git a/sncf/api/views.py b/sncf/api/views.py index 5ded803..8b92853 100644 --- a/sncf/api/views.py +++ b/sncf/api/views.py @@ -15,7 +15,9 @@ from sncfgtfs.models import Agency, Stop, Route, Trip, StopTime, Calendar, Calen Transfer, FeedInfo, TripUpdate, StopTimeUpdate CACHE_CONTROL = cache_control(max_age=7200) -LAST_MODIFIED = last_modified(lambda *args, **kwargs: datetime.fromisoformat(FeedInfo.objects.get().version)) +LAST_MODIFIED = last_modified(lambda *args, **kwargs: datetime.fromisoformat( + FeedInfo.objects.get(publisher_name="SNCF_default").version)) +LOOKUP_VALUE_REGEX = r"[\w.: |-]+" @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -25,6 +27,7 @@ class AgencyViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = AgencySerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -35,6 +38,7 @@ class StopViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = [DjangoFilterBackend, SearchFilter] filterset_fields = '__all__' search_fields = ['name',] + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -44,6 +48,7 @@ class RouteViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = RouteSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -53,6 +58,7 @@ class TripViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = TripSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -64,6 +70,7 @@ class StopTimeViewSet(viewsets.ReadOnlyModelViewSet): filterset_fields = '__all__' ordering_fields = ['arrival_time', 'departure_time', 'stop_sequence', ] ordering = ['stop_sequence', ] + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -73,6 +80,7 @@ class CalendarViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = CalendarSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -82,6 +90,7 @@ class CalendarDateViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = CalendarDateSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -90,6 +99,7 @@ class TransferViewSet(viewsets.ReadOnlyModelViewSet): queryset = Transfer.objects.all() serializer_class = TransferSerializer filter_backends = [DjangoFilterBackend] + lookup_value_regex = LOOKUP_VALUE_REGEX @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @@ -99,6 +109,7 @@ class FeedInfoViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = FeedInfoSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX class TripUpdateViewSet(viewsets.ReadOnlyModelViewSet): @@ -106,6 +117,7 @@ class TripUpdateViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = TripUpdateSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX class StopTimeUpdateViewSet(viewsets.ReadOnlyModelViewSet): @@ -113,6 +125,7 @@ class StopTimeUpdateViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = StopTimeUpdateSerializer filter_backends = [DjangoFilterBackend] filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): @@ -140,7 +153,7 @@ class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): stop = Stop.objects.get(id=stop_id) stops = Stop.objects.filter(Q(id=stop_id) | Q(parent_station=stop_id)) - if stop.location_type == 0: + if stop.location_type == 0 and stop.parent_station_id is not None: stops |= Stop.objects.filter(parent_station=stop.parent_station_id) stop_filter = Q(stop__in=stops.values_list('id', flat=True)) elif stop_name: @@ -201,7 +214,7 @@ class NextArrivalsViewSet(viewsets.ReadOnlyModelViewSet): stop = Stop.objects.get(id=stop_id) stops = Stop.objects.filter(Q(id=stop_id) | Q(parent_station=stop_id)) - if stop.location_type == 0: + if stop.location_type == 0 and stop.parent_station_id is not None: stops |= Stop.objects.filter(parent_station=stop.parent_station_id) stop_filter = Q(stop__in=stops.values_list('id', flat=True)) elif stop_name: diff --git a/sncfgtfs/admin.py b/sncfgtfs/admin.py index 2345f0c..fcc9600 100644 --- a/sncfgtfs/admin.py +++ b/sncfgtfs/admin.py @@ -44,7 +44,7 @@ class AgencyAdmin(admin.ModelAdmin): @admin.register(Stop) class StopAdmin(admin.ModelAdmin): list_display = ('name', 'id', 'lat', 'lon', 'location_type',) - list_filter = ('location_type',) + list_filter = ('location_type', 'transport_type',) search_fields = ('name', 'id',) ordering = ('name',) autocomplete_fields = ('parent_station',) @@ -52,8 +52,8 @@ class StopAdmin(admin.ModelAdmin): @admin.register(Route) class RouteAdmin(admin.ModelAdmin): - list_display = ('long_name', 'short_name', 'id', 'type',) - list_filter = ('type',) + list_display = ('short_name', 'long_name', 'id', 'type',) + list_filter = ('transport_type', 'type', 'agency',) search_fields = ('long_name', 'short_name', 'id',) ordering = ('long_name',) autocomplete_fields = ('agency',) diff --git a/sncfgtfs/management/commands/update_sncf_gtfs.py b/sncfgtfs/management/commands/update_sncf_gtfs.py index 5972216..207a0e4 100644 --- a/sncfgtfs/management/commands/update_sncf_gtfs.py +++ b/sncfgtfs/management/commands/update_sncf_gtfs.py @@ -17,6 +17,10 @@ class Command(BaseCommand): "IC": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-intercites-gtfs-last.zip", "TER": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-ter-gtfs-last.zip", "TN": "https://eu.ftp.opendatasoft.com/sncf/gtfs/transilien-gtfs.zip", + # "ES": "https://www.data.gouv.fr/fr/datasets/r/9089b550-696e-4ae0-87b5-40ea55a14292", + # "TI": "https://www.data.gouv.fr/fr/datasets/r/4d1dd21a-b061-47ac-9514-57ffcc09b4a5", + # "RENFE": "https://ssl.renfe.com/gtransit/Fichero_AV_LD/google_transit.zip", + # "OBB": "https://static.oebb.at/open-data/soll-fahrplan-gtfs/GTFS_OP_2024_obb.zip", } def add_arguments(self, parser): @@ -35,10 +39,13 @@ class Command(BaseCommand): if not FeedInfo.objects.exists(): last_update_date = "1970-01-01" else: - last_update_date = FeedInfo.objects.get().version + last_update_date = FeedInfo.objects.get(publisher_name='SNCF_default').version for url in self.GTFS_FEEDS.values(): - last_modified = requests.head(url).headers["Last-Modified"] + resp = requests.head(url) + if "Last-Modified" not in resp.headers: + continue + last_modified = resp.headers["Last-Modified"] last_modified = datetime.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z") if last_modified.date().isoformat() > last_update_date: break @@ -54,15 +61,22 @@ class Command(BaseCommand): for transport_type, feed_url in self.GTFS_FEEDS.items(): self.stdout.write(f"Downloading {transport_type} GTFS feed...") with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile: + def read_file(filename): + lines = zipfile.read(filename).decode().replace('\ufeff', '').splitlines() + return [line.strip() for line in lines] + agencies = [] - for agency_dict in csv.DictReader(zipfile.read("agency.txt").decode().splitlines()): + for agency_dict in csv.DictReader(read_file("agency.txt")): agency_dict: dict + if transport_type == "ES" \ + and agency_dict['agency_id'] != 'ES' and agency_dict['agency_id'] != 'ER': + continue agency = Agency( id=agency_dict['agency_id'], name=agency_dict['agency_name'], url=agency_dict['agency_url'], timezone=agency_dict['agency_timezone'], - lang=agency_dict['agency_lang'], + lang=agency_dict.get('agency_lang', "fr"), phone=agency_dict.get('agency_phone', ""), email=agency_dict.get('agency_email', ""), ) @@ -75,23 +89,28 @@ class Command(BaseCommand): agencies.clear() stops = [] - for stop_dict in csv.DictReader(zipfile.read("stops.txt").decode().splitlines()): + for stop_dict in csv.DictReader(read_file("stops.txt")): stop_dict: dict + stop_id = stop_dict['stop_id'] + if transport_type in ["ES", "TI", "RENFE"]: + stop_id = f"{transport_type}-{stop_id}" + stop = Stop( - id=stop_dict["stop_id"], + id=stop_id, name=stop_dict['stop_name'], - desc=stop_dict['stop_desc'], + desc=stop_dict.get('stop_desc', ""), lat=stop_dict['stop_lat'], lon=stop_dict['stop_lon'], - zone_id=stop_dict['zone_id'], - url=stop_dict['stop_url'], - location_type=stop_dict['location_type'], - parent_station_id=stop_dict['parent_station'] or None + zone_id=stop_dict.get('zone_id', ""), + url=stop_dict.get('stop_url', ""), + location_type=stop_dict.get('location_type', 1) or 1, + parent_station_id=stop_dict.get('parent_station', None) or None if last_update_date != "1970-01-01" or transport_type != "TN" else None, timezone=stop_dict.get('stop_timezone', ""), wheelchair_boarding=stop_dict.get('wheelchair_boarding', 0), level_id=stop_dict.get('level_id', ""), platform_code=stop_dict.get('platform_code', ""), + transport_type=transport_type, ) stops.append(stop) @@ -100,7 +119,8 @@ class Command(BaseCommand): update_conflicts=True, update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url', 'location_type', 'parent_station_id', 'timezone', - 'wheelchair_boarding', 'level_id', 'platform_code'], + 'wheelchair_boarding', 'level_id', 'platform_code', + 'transport_type'], unique_fields=['id']) stops.clear() if stops and not dry_run: @@ -108,23 +128,27 @@ class Command(BaseCommand): update_conflicts=True, update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url', 'location_type', 'parent_station_id', 'timezone', - 'wheelchair_boarding', 'level_id', 'platform_code'], + 'wheelchair_boarding', 'level_id', 'platform_code', + 'transport_type'], unique_fields=['id']) stops.clear() routes = [] - for route_dict in csv.DictReader(zipfile.read("routes.txt").decode().splitlines()): + for route_dict in csv.DictReader(read_file("routes.txt")): route_dict: dict + route_id = route_dict['route_id'] + if transport_type == "TI": + route_id = f"{transport_type}-{route_id}" route = Route( - id=route_dict['route_id'], + id=route_id, agency_id=route_dict['agency_id'], short_name=route_dict['route_short_name'], long_name=route_dict['route_long_name'], - desc=route_dict['route_desc'], + desc=route_dict.get('route_desc', ""), type=route_dict['route_type'], - url=route_dict['route_url'], - color=route_dict['route_color'], - text_color=route_dict['route_text_color'], + url=route_dict.get('route_url', ""), + color=route_dict.get('route_color', ""), + text_color=route_dict.get('route_text_color', ""), transport_type=transport_type, ) routes.append(route) @@ -141,14 +165,15 @@ class Command(BaseCommand): Route.objects.bulk_create(routes, update_conflicts=True, update_fields=['agency_id', 'short_name', 'long_name', 'desc', - 'type', 'url', 'color', 'text_color'], + 'type', 'url', 'color', 'text_color', + 'transport_type'], unique_fields=['id']) routes.clear() calendar_ids = [] if "calendar.txt" in zipfile.namelist(): calendars = [] - for calendar_dict in csv.DictReader(zipfile.read("calendar.txt").decode().splitlines()): + for calendar_dict in csv.DictReader(read_file("calendar.txt")): calendar_dict: dict calendar = Calendar( id=f"{transport_type}-{calendar_dict['service_id']}", @@ -184,7 +209,7 @@ class Command(BaseCommand): calendars = [] calendar_dates = [] - for calendar_date_dict in csv.DictReader(zipfile.read("calendar_dates.txt").decode().splitlines()): + for calendar_date_dict in csv.DictReader(read_file("calendar_dates.txt")): calendar_date_dict: dict calendar_date = CalendarDate( id=f"{transport_type}-{calendar_date_dict['service_id']}-{calendar_date_dict['date']}", @@ -235,23 +260,31 @@ class Command(BaseCommand): calendar_dates.clear() trips = [] - for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()): + for trip_dict in csv.DictReader(read_file("trips.txt")): trip_dict: dict trip_id = trip_dict['trip_id'] - if transport_type != "TN": + route_id = trip_dict['route_id'] + if transport_type in ["TGV", "IC", "TER"]: trip_id, last_update = trip_id.split(':', 1) last_update = datetime.fromisoformat(last_update) + elif transport_type in ["ES", "RENFE"]: + trip_id = f"{transport_type}-{trip_id}" + last_update = None + elif transport_type == "TI": + trip_id = f"{transport_type}-{trip_id}" + route_id = f"{transport_type}-{route_id}" + last_update = None else: last_update = None trip = Trip( id=trip_id, - route_id=trip_dict['route_id'], + route_id=route_id, service_id=f"{transport_type}-{trip_dict['service_id']}", - headsign=trip_dict['trip_headsign'], + headsign=trip_dict.get('trip_headsign', ""), short_name=trip_dict.get('trip_short_name', ""), - direction_id=trip_dict['direction_id'] or None, - block_id=trip_dict['block_id'], - shape_id=trip_dict['shape_id'], + direction_id=trip_dict.get('direction_id', None) or None, + block_id=trip_dict.get('block_id', ""), + shape_id=trip_dict.get('shape_id', ""), wheelchair_accessible=trip_dict.get('wheelchair_accessible', None), bikes_allowed=trip_dict.get('bikes_allowed', None), last_update=last_update, @@ -278,26 +311,49 @@ class Command(BaseCommand): all_trips.extend(trips) stop_times = [] - for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()): + for stop_time_dict in csv.DictReader(read_file("stop_times.txt")): stop_time_dict: dict + stop_id = stop_time_dict['stop_id'] + if transport_type in ["ES", "TI", "RENFE"]: + stop_id = f"{transport_type}-{stop_id}" + trip_id = stop_time_dict['trip_id'] - if transport_type != "TN": + if transport_type in ["TGV", "IC", "TER"]: trip_id = trip_id.split(':', 1)[0] + elif transport_type in ["ES", "TI", "RENFE"]: + trip_id = f"{transport_type}-{trip_id}" + arr_time = stop_time_dict['arrival_time'] - arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:]) + arr_h, arr_m, arr_s = map(int, arr_time.split(':')) + arr_time = arr_h * 3600 + arr_m * 60 + arr_s dep_time = stop_time_dict['departure_time'] - dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:]) + dep_h, dep_m, dep_s = map(int, dep_time.split(':')) + dep_time = dep_h * 3600 + dep_m * 60 + dep_s + + pickup_type = stop_time_dict.get('pickup_type', 0) + drop_off_type = stop_time_dict.get('drop_off_type', 0) + if transport_type in ["ES", "RENFE", "OBB"]: + if stop_time_dict['stop_sequence'] == "1": + drop_off_type = 1 + elif arr_time == dep_time: + pickup_type = 1 + elif transport_type == "TI": + if stop_time_dict['stop_sequence'] == "0": + drop_off_type = 1 + elif arr_time == dep_time: + pickup_type = 1 + st = StopTime( - id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}", + id=f"{trip_id}-{stop_id}", trip_id=trip_id, arrival_time=timedelta(seconds=arr_time), departure_time=timedelta(seconds=dep_time), - stop_id=stop_time_dict['stop_id'], + stop_id=stop_id, stop_sequence=stop_time_dict['stop_sequence'], - stop_headsign=stop_time_dict['stop_headsign'], - pickup_type=stop_time_dict['pickup_type'], - drop_off_type=stop_time_dict['drop_off_type'], + stop_headsign=stop_time_dict.get('stop_headsign', ""), + pickup_type=pickup_type, + drop_off_type=drop_off_type, timepoint=stop_time_dict.get('timepoint', None), ) stop_times.append(st) @@ -319,42 +375,49 @@ class Command(BaseCommand): unique_fields=['id']) stop_times.clear() - transfers = [] - for transfer_dict in csv.DictReader(zipfile.read("transfers.txt").decode().splitlines()): - transfer_dict: dict - transfer = Transfer( - id=f"{transfer_dict['from_stop_id']}-{transfer_dict['to_stop_id']}", - from_stop_id=transfer_dict['from_stop_id'], - to_stop_id=transfer_dict['to_stop_id'], - transfer_type=transfer_dict['transfer_type'], - min_transfer_time=transfer_dict['min_transfer_time'], - ) - transfers.append(transfer) + if "transfers.txt" in zipfile.namelist(): + transfers = [] + for transfer_dict in csv.DictReader(read_file("transfers.txt")): + transfer_dict: dict + from_stop_id = transfer_dict['from_stop_id'] + to_stop_id = transfer_dict['to_stop_id'] + if transport_type in ["ES", "RENFE", "OBB"]: + from_stop_id = f"{transport_type}-{from_stop_id}" + to_stop_id = f"{transport_type}-{to_stop_id}" - if len(transfers) >= bulk_size and not dry_run: + transfer = Transfer( + id=f"{from_stop_id}-{to_stop_id}", + from_stop_id=transfer_dict['from_stop_id'], + to_stop_id=transfer_dict['to_stop_id'], + transfer_type=transfer_dict['transfer_type'], + min_transfer_time=transfer_dict['min_transfer_time'], + ) + transfers.append(transfer) + + if len(transfers) >= bulk_size and not dry_run: + Transfer.objects.bulk_create(transfers, + update_conflicts=True, + update_fields=['transfer_type', 'min_transfer_time'], + unique_fields=['id']) + transfers.clear() + + if transfers and not dry_run: Transfer.objects.bulk_create(transfers, update_conflicts=True, update_fields=['transfer_type', 'min_transfer_time'], unique_fields=['id']) transfers.clear() - if transfers and not dry_run: - Transfer.objects.bulk_create(transfers, - update_conflicts=True, - update_fields=['transfer_type', 'min_transfer_time'], - unique_fields=['id']) - transfers.clear() - if "feed_info.txt" in zipfile.namelist() and not dry_run: - for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()): + for feed_info_dict in csv.DictReader(read_file("feed_info.txt")): feed_info_dict: dict FeedInfo.objects.update_or_create( publisher_name=feed_info_dict['feed_publisher_name'], defaults=dict( publisher_url=feed_info_dict['feed_publisher_url'], lang=feed_info_dict['feed_lang'], - start_date=feed_info_dict['feed_start_date'], - end_date=feed_info_dict['feed_end_date'], - version=feed_info_dict['feed_version'], + start_date=feed_info_dict.get('feed_start_date', datetime.now().date()), + end_date=feed_info_dict.get('feed_end_date', datetime.now().date()), + version=feed_info_dict.get('feed_version', 1), ) ) diff --git a/sncfgtfs/migrations/0001_initial.py b/sncfgtfs/migrations/0001_initial.py index 9fd72bc..aa1accb 100644 --- a/sncfgtfs/migrations/0001_initial.py +++ b/sncfgtfs/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.1 on 2024-02-09 21:55 +# Generated by Django 5.0.1 on 2024-02-10 16:30 import django.db.models.deletion from django.db import migrations, models @@ -22,12 +22,7 @@ class Migration(migrations.Migration): verbose_name="Agency ID", ), ), - ( - "name", - models.CharField( - max_length=255, unique=True, verbose_name="Agency name" - ), - ), + ("name", models.CharField(max_length=255, verbose_name="Agency name")), ("url", models.URLField(verbose_name="Agency URL")), ( "timezone", @@ -87,6 +82,10 @@ class Migration(migrations.Migration): ("TER", "TER"), ("IC", "Intercités"), ("TN", "Transilien"), + ("ES", "Eurostar"), + ("TI", "Trenitalia"), + ("RENFE", "Renfe"), + ("OBB", "ÖBB"), ], max_length=255, verbose_name="Transport type", @@ -234,6 +233,10 @@ class Migration(migrations.Migration): ("TER", "TER"), ("IC", "Intercités"), ("TN", "Transilien"), + ("ES", "Eurostar"), + ("TI", "Trenitalia"), + ("RENFE", "Renfe"), + ("OBB", "ÖBB"), ], max_length=255, verbose_name="Transport type", @@ -330,6 +333,23 @@ class Migration(migrations.Migration): blank=True, max_length=255, verbose_name="Platform code" ), ), + ( + "transport_type", + models.CharField( + choices=[ + ("TGV", "TGV"), + ("TER", "TER"), + ("IC", "Intercités"), + ("TN", "Transilien"), + ("ES", "Eurostar"), + ("TI", "Trenitalia"), + ("RENFE", "Renfe"), + ("OBB", "ÖBB"), + ], + max_length=255, + verbose_name="Transport type", + ), + ), ( "parent_station", models.ForeignKey( diff --git a/sncfgtfs/models.py b/sncfgtfs/models.py index 0410581..5bfae6f 100644 --- a/sncfgtfs/models.py +++ b/sncfgtfs/models.py @@ -7,6 +7,10 @@ class TransportType(models.TextChoices): TER = "TER", _("TER") INTERCITES = "IC", _("Intercités") TRANSILIEN = "TN", _("Transilien") + EUROSTAR = "ES", _("Eurostar") + TRENITALIA = "TI", _("Trenitalia") + RENFE = "RENFE", _("Renfe") + OBB = "OBB", _("ÖBB") class LocationType(models.IntegerChoices): @@ -84,7 +88,6 @@ class Agency(models.Model): name = models.CharField( max_length=255, - unique=True, verbose_name=_("Agency name"), ) @@ -206,6 +209,12 @@ class Stop(models.Model): blank=True, ) + transport_type = models.CharField( + max_length=255, + verbose_name=_("Transport type"), + choices=TransportType, + ) + @property def stop_type(self): train_type = self.id.split('StopPoint:OCE')[1].split('-')[0]