From bd8d39fc1e71a13a26560bdccdd88555e1d6ab35 Mon Sep 17 00:00:00 2001 From: Emmy D'Anello Date: Mon, 12 Aug 2024 20:49:17 +0200 Subject: [PATCH] Visual prototype to filter routes --- trainvel-front/package-lock.json | 26 +++ trainvel-front/package.json | 1 + trainvel-front/src/Station.js | 2 + trainvel-front/src/TrainsTable.js | 2 + trainvel-front/src/TripsFilter.js | 165 +++++++++++++++++ trainvel/api/serializers.py | 10 +- trainvel/api/views.py | 54 +++++- trainvel/gtfs/admin.py | 1 + trainvel/gtfs/fixtures/gtfs_feeds.json | 90 ++++++++-- .../gtfs/management/commands/augment_data.py | 20 +++ ...003_gtfsfeed_categorize_routes_and_more.py | 72 ++++++++ ...rip_route_name_trip_route_type_and_more.py | 39 ++++ trainvel/gtfs/models.py | 167 ++++++++++++++++++ 13 files changed, 630 insertions(+), 19 deletions(-) create mode 100644 trainvel-front/src/TripsFilter.js create mode 100644 trainvel/gtfs/management/commands/augment_data.py create mode 100644 trainvel/gtfs/migrations/0003_gtfsfeed_categorize_routes_and_more.py create mode 100644 trainvel/gtfs/migrations/0004_trip_long_distance_trip_route_name_trip_route_type_and_more.py diff --git a/trainvel-front/package-lock.json b/trainvel-front/package-lock.json index 1823fd6..7b0738b 100644 --- a/trainvel-front/package-lock.json +++ b/trainvel-front/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@emotion/react": "^11.11.3", "@emotion/styled": "^11.11.0", + "@mui/icons-material": "^5.15.17", "@mui/material": "^5.15.6", "@mui/x-date-pickers": "^6.19.2", "@tanstack/query-sync-storage-persister": "^5.18.0", @@ -3476,6 +3477,31 @@ "url": "https://opencollective.com/mui-org" } }, + "node_modules/@mui/icons-material": { + "version": "5.15.17", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-5.15.17.tgz", + "integrity": "sha512-xVzl2De7IY36s/keHX45YMiCpsIx3mNv2xwDgtBkRSnZQtVk+Gqufwj1ktUxEyjzEhBl0+PiNJqYC31C+n1n6A==", + "dependencies": { + "@babel/runtime": "^7.23.9" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@mui/material": "^5.0.0", + "@types/react": "^17.0.0 || ^18.0.0", + "react": "^17.0.0 || ^18.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/@mui/material": { "version": "5.15.6", "resolved": "https://registry.npmjs.org/@mui/material/-/material-5.15.6.tgz", diff --git a/trainvel-front/package.json b/trainvel-front/package.json index 1297d19..c01d84e 100644 --- a/trainvel-front/package.json +++ b/trainvel-front/package.json @@ -5,6 +5,7 @@ "dependencies": { "@emotion/react": "^11.11.3", "@emotion/styled": "^11.11.0", + "@mui/icons-material": "^5.15.17", "@mui/material": "^5.15.6", "@mui/x-date-pickers": "^6.19.2", "@tanstack/query-sync-storage-persister": "^5.18.0", diff --git a/trainvel-front/src/Station.js b/trainvel-front/src/Station.js index c35b420..3810ad7 100644 --- a/trainvel-front/src/Station.js +++ b/trainvel-front/src/Station.js @@ -1,5 +1,6 @@ import {useNavigate, useParams, useSearchParams} from "react-router-dom" import TrainsTable from "./TrainsTable" +import TripsFilter from "./TripsFilter" import {useState} from "react"; import {Box, Button, FormLabel} from "@mui/material"; import {DatePicker, TimePicker} from "@mui/x-date-pickers"; @@ -67,6 +68,7 @@ function Station() {
+
diff --git a/trainvel-front/src/TrainsTable.js b/trainvel-front/src/TrainsTable.js index 947e3c1..aabbe6d 100644 --- a/trainvel-front/src/TrainsTable.js +++ b/trainvel-front/src/TrainsTable.js @@ -257,6 +257,8 @@ function getTrainType(train, trip, route) { if (trip.short_name?.startsWith("NJ")) return "NJ" return "ÖBB" + case "CH-ALL": + return route.desc default: return trip.short_name?.split(" ")[0] } diff --git a/trainvel-front/src/TripsFilter.js b/trainvel-front/src/TripsFilter.js new file mode 100644 index 0000000..01ddb24 --- /dev/null +++ b/trainvel-front/src/TripsFilter.js @@ -0,0 +1,165 @@ +import {useState} from "react" +import { + Box, Button, + Checkbox, Chip, FormControl, + FormControlLabel, + InputLabel, MenuItem, OutlinedInput, Select +} from "@mui/material" +import DirectionsBusTwoToneIcon from '@mui/icons-material/DirectionsBusTwoTone' +import SubwayTwoToneIcon from '@mui/icons-material/SubwayTwoTone' +import TrainTwoToneIcon from '@mui/icons-material/TrainTwoTone' +import TramTwoToneIcon from '@mui/icons-material/TramTwoTone' + +function TripsFilter() { + const [transportModeFilter, setTransportModeFilter] = useState( + {longDistanceTrain: true, regionalTrain: true, metro: true, tram: true, bus: true}) + const transportModeNames = { + train: "Trains", + longDistanceTrain: "Trains longue distance", + regionalTrain: "Trains régionaux", + metro: "Métro", + tram: "Tram", + bus: "Bus", + } + + const trainCheckbox = <> + + + setTransportModeFilter( + {...transportModeFilter, longDistanceTrain: event.target.checked, regionalTrain: event.target.checked})} + onClick={(event) => event.stopPropagation()} + /> + + + const longDistanceTrainCheckbox = <> + + + setTransportModeFilter({...transportModeFilter, longDistanceTrain: event.target.checked})} /> + + + const regionalTrainCheckbox = <> + + + setTransportModeFilter({...transportModeFilter, regionalTrain: event.target.checked})} /> + + + const metroCheckbox = <> + + + setTransportModeFilter({...transportModeFilter, metro: event.target.checked})} /> + + + const tramCheckbox = <> + + + setTransportModeFilter({...transportModeFilter, tram: event.target.checked})} /> + + + const busCheckbox = <> + + + setTransportModeFilter({...transportModeFilter, bus: event.target.checked})} /> + + + // TODO Fetch routes that are accessible from one stop + // For now, we have the tram and bus routes accessible in Strasbourg main station + const routesList = [ + {name: "Tous"}, + {name: "A", bgColor: "#E10D19", color: "#FFFFFF"}, + {name: "C", bgColor: "#F29400", color: "#FFFFFF"}, + {name: "D", bgColor: "#009933", color: "#FFFFFF"}, + {name: "G", bgColor: "#F6C900", color: "#000000"}, + {name: "H", bgColor: "#A62341", color: "#FFFFFF"}, + {name: "2", bgColor: "#FF0000", color: "#FFFFFF"}, + {name: "10", bgColor: "#FFAA00", color: "#000000"}, + ] + const routesDict = {} + for (const route of routesList) { + routesDict[route.name] = route + } + + const [selectedRoutes, setSelectedRoutes] = useState(["Tous"]) + + return <> +

Filtres

+ + + Mode de transport + + + + + Ligne + + + + + + +} + +export default TripsFilter diff --git a/trainvel/api/serializers.py b/trainvel/api/serializers.py index 4b97ac7..8ab0330 100644 --- a/trainvel/api/serializers.py +++ b/trainvel/api/serializers.py @@ -1,8 +1,8 @@ from rest_framework import serializers from trainvel.core.models import Station -from trainvel.gtfs.models import Agency, Stop, Route, Trip, StopTime, Calendar, CalendarDate, \ - Transfer, FeedInfo, TripUpdate, StopTimeUpdate +from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, \ + Stop, StopTime, StopTimeUpdate, Transfer, Trip, TripUpdate class StationSerializer(serializers.ModelSerializer): @@ -13,6 +13,12 @@ class StationSerializer(serializers.ModelSerializer): 'country_hint', 'main_station_hint',) +class GTFSFeedSerializer(serializers.ModelSerializer): + class Meta: + model = GTFSFeed + fields = '__all__' + + class AgencySerializer(serializers.ModelSerializer): class Meta: model = Agency diff --git a/trainvel/api/views.py b/trainvel/api/views.py index 4289c70..887b296 100644 --- a/trainvel/api/views.py +++ b/trainvel/api/views.py @@ -8,13 +8,12 @@ from django_filters.rest_framework import DjangoFilterBackend from rest_framework import viewsets from rest_framework.filters import OrderingFilter, SearchFilter -from trainvel.api.serializers import AgencySerializer, StopSerializer, RouteSerializer, TripSerializer, \ - StopTimeSerializer, CalendarSerializer, CalendarDateSerializer, TransferSerializer, \ - FeedInfoSerializer, TripUpdateSerializer, StopTimeUpdateSerializer, StationSerializer +from trainvel.api.serializers import AgencySerializer, CalendarDateSerializer, CalendarSerializer, \ + FeedInfoSerializer, GTFSFeedSerializer, RouteSerializer, StationSerializer, StopSerializer, StopTimeSerializer, \ + StopTimeUpdateSerializer, TransferSerializer, TripSerializer, TripUpdateSerializer from trainvel.core.models import Station from trainvel.gtfs.models import Agency, Calendar, CalendarDate, FeedInfo, GTFSFeed, Route, Stop, StopTime, \ - StopTimeUpdate, \ - Transfer, Trip, TripUpdate, PickupType + StopTimeUpdate, Transfer, Trip, TripUpdate, PickupType CACHE_CONTROL = cache_control(max_age=30) LAST_MODIFIED = last_modified(lambda *args, **kwargs: GTFSFeed.objects.order_by('-last_modified').first().last_modified) @@ -34,6 +33,16 @@ class StationViewSet(viewsets.ReadOnlyModelViewSet): lookup_value_regex = LOOKUP_VALUE_REGEX +@method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) +@method_decorator(name='retrieve', decorator=[CACHE_CONTROL, LAST_MODIFIED]) +class GTFSFeedViewSet(viewsets.ReadOnlyModelViewSet): + queryset = GTFSFeed.objects.all() + serializer_class = GTFSFeedSerializer + filter_backends = [DjangoFilterBackend] + filterset_fields = '__all__' + lookup_value_regex = LOOKUP_VALUE_REGEX + + @method_decorator(name='list', decorator=[CACHE_CONTROL, LAST_MODIFIED]) @method_decorator(name='retrieve', decorator=[CACHE_CONTROL, LAST_MODIFIED]) class AgencyViewSet(viewsets.ReadOnlyModelViewSet): @@ -145,7 +154,6 @@ class StopTimeUpdateViewSet(viewsets.ReadOnlyModelViewSet): class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): queryset = StopTime.objects.none() serializer_class = StopTimeSerializer - filter_backends = [DjangoFilterBackend] def get_queryset(self): now = datetime.now() @@ -167,10 +175,21 @@ class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): near_stops = station.get_near_stops() stop_filter = Q(stop_id__in=near_stops.values_list('id', flat=True)) + excluded_agencies = ~Q(trip__route__gtfs_feed__excluded_agencies=F('trip__route__agency_id')) + not_last_stop = ~Q(stop_sequence=StopTime.objects.filter(trip_id=OuterRef('trip_id')) .filter(pickup_type=PickupType.REGULAR) .order_by('-stop_sequence')[:1].values_list('stop_sequence')) + trip_filter = Q() + if self.request.query_params.get('route_name', None): + trip_filter &= Q(trip__route_name__in=self.request.query_params.get('route_name').split(',')) + if self.request.query_params.get('transport_type', None): + trip_filter &= Q(trip__route__type__in=self.request.query_params.get('transport_type').split(',')) + if self.request.query_params.get('long_distance', None) is not None: + long_distance = str(self.request.query_params.get('long_distance')) == 'true' + trip_filter &= Q(trip__long_distance=long_distance) + def calendar_filter(d: date): return Q(trip__service_id__in=CalendarDate.objects.filter(date=d, exception_type=1) .values_list('service_id')) \ @@ -199,7 +218,9 @@ class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): return Exists(stop_time_update_qs(d).filter(Q(schedule_relationship=1) | Q(schedule_relationship=3))) qs_today = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_last_stop) \ + .filter(trip_filter) \ .annotate(departure_time_real=departure_time_real(query_date)) \ .filter(departure_time_real__gte=query_time) \ .filter(Q(pickup_type=PickupType.REGULAR) | canceled_filter(query_date)) \ @@ -208,7 +229,9 @@ class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): .annotate(departure_time_24h=F('departure_time')) qs_yesterday = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_last_stop) \ + .filter(trip_filter) \ .annotate(departure_time_real=departure_time_real(query_date)) \ .filter(departure_time_real__gte=time_yesterday) \ .filter(Q(pickup_type=PickupType.REGULAR) | canceled_filter(yesterday)) \ @@ -217,7 +240,9 @@ class NextDeparturesViewSet(viewsets.ReadOnlyModelViewSet): .annotate(departure_time_24h=F('departure_time') - timedelta(days=1)) qs_tomorrow = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_last_stop) \ + .filter(trip_filter) \ .annotate(departure_time_real=departure_time_real(query_date)) \ .filter(departure_time_real__gte=timedelta(0)) \ .filter(Q(pickup_type=PickupType.REGULAR) | canceled_filter(tomorrow)) \ @@ -254,10 +279,21 @@ class NextArrivalsViewSet(viewsets.ReadOnlyModelViewSet): near_stops = station.get_near_stops() stop_filter = Q(stop_id__in=near_stops.values_list('id', flat=True)) + excluded_agencies = ~Q(trip__route__gtfs_feed__excluded_agencies=F('trip__route__agency_id')) + not_first_stop = ~Q(stop_sequence=StopTime.objects.filter(trip_id=OuterRef('trip_id')) .filter(drop_off_type=PickupType.REGULAR) .order_by('stop_sequence')[:1].values_list('stop_sequence')) + trip_filter = Q() + if self.request.query_params.get('route_name', None): + trip_filter &= Q(trip__route_name__in=self.request.query_params.get('route_name').split(',')) + if self.request.query_params.get('transport_type', None): + trip_filter &= Q(trip__route__type__in=self.request.query_params.get('transport_type').split(',')) + if self.request.query_params.get('long_distance', None) is not None: + long_distance = str(self.request.query_params.get('long_distance')) == 'true' + trip_filter &= Q(trip__long_distance=long_distance) + def calendar_filter(d: date): return Q(trip__service_id__in=CalendarDate.objects.filter(date=d, exception_type=1) .values_list('service_id')) \ @@ -286,7 +322,9 @@ class NextArrivalsViewSet(viewsets.ReadOnlyModelViewSet): return Exists(stop_time_update_qs(d).filter(Q(schedule_relationship=1) | Q(schedule_relationship=3))) qs_today = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_first_stop) \ + .filter(trip_filter) \ .annotate(arrival_time_real=arrival_time_real(query_date)) \ .filter(arrival_time_real__gte=query_time) \ .filter(Q(drop_off_type=PickupType.REGULAR) | canceled_filter(query_date)) \ @@ -295,7 +333,9 @@ class NextArrivalsViewSet(viewsets.ReadOnlyModelViewSet): .annotate(arrival_time_24h=F('arrival_time')) qs_yesterday = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_first_stop) \ + .filter(trip_filter) \ .annotate(arrival_time_real=arrival_time_real(yesterday)) \ .filter(arrival_time_real__gte=time_yesterday) \ .filter(Q(drop_off_type=PickupType.REGULAR) | canceled_filter(yesterday)) \ @@ -304,7 +344,9 @@ class NextArrivalsViewSet(viewsets.ReadOnlyModelViewSet): .annotate(arrival_time_24h=F('arrival_time') - timedelta(days=1)) qs_tomorrow = StopTime.objects.filter(stop_filter) \ + .filter(excluded_agencies) \ .filter(not_first_stop) \ + .filter(trip_filter) \ .annotate(arrival_time_real=arrival_time_real(tomorrow)) \ .filter(arrival_time_real__gte=timedelta(0)) \ .filter(Q(drop_off_type=PickupType.REGULAR) | canceled_filter(tomorrow)) \ diff --git a/trainvel/gtfs/admin.py b/trainvel/gtfs/admin.py index e4e0f86..12475d3 100644 --- a/trainvel/gtfs/admin.py +++ b/trainvel/gtfs/admin.py @@ -57,6 +57,7 @@ class GTFSFeedAdmin(admin.ModelAdmin): list_filter = ('country', 'last_modified',) search_fields = ('name', 'code',) readonly_fields = ('code',) + autocomplete_fields = ('excluded_agencies',) @admin.register(Agency) diff --git a/trainvel/gtfs/fixtures/gtfs_feeds.json b/trainvel/gtfs/fixtures/gtfs_feeds.json index 5274895..5cd894e 100644 --- a/trainvel/gtfs/fixtures/gtfs_feeds.json +++ b/trainvel/gtfs/fixtures/gtfs_feeds.json @@ -6,7 +6,13 @@ "name": "SNCF - TGV", "country": "FR", "feed_url": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export_gtfs_voyages.zip", - "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-tgv-gtfs-rt-trip-updates" + "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-tgv-gtfs-rt-trip-updates", + "categorize_routes": false, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "SNCF-&stop:FR-SNCF-TGV-StopPoint:OCE([\\w\\s]+)-\\d+", + "trip_number_regex": "trip_headsign:(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -16,7 +22,13 @@ "name": "SNCF - Intercités", "country": "FR", "feed_url": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-intercites-gtfs-last.zip", - "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-ic-gtfs-rt-trip-updates" + "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-ic-gtfs-rt-trip-updates", + "categorize_routes": false, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "SNCF-&stop:FR-SNCF-IC-StopPoint:OCE([\\w\\s]+)-\\d+", + "trip_number_regex": "trip_headsign:(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -26,7 +38,13 @@ "name": "SNCF - TER", "country": "FR", "feed_url": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-ter-gtfs-last.zip", - "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-ter-gtfs-rt-trip-updates" + "rt_feed_url": "https://proxy.transport.data.gouv.fr/resource/sncf-ter-gtfs-rt-trip-updates", + "categorize_routes": false, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "SNCF-&stop:FR-SNCF-TER-StopPoint:OCE([\\w\\s]+)-\\d+", + "trip_number_regex": "trip_headsign:(\\d+)", + "long_distance_regex": "^$", + "excluded_agencies": [] } }, { @@ -36,7 +54,13 @@ "name": "Île-de-France Mobilités", "country": "FR", "feed_url": "https://eu.ftp.opendatasoft.com/stif/GTFS/IDFM-gtfs.zip", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w]+)", + "route_type_regex": "IDFM", + "trip_number_regex": "trip_short_name:([\\d\\w-]*)§", + "long_distance_regex": "^$", + "excluded_agencies": [] } }, { @@ -46,7 +70,13 @@ "name": "Compagnie des Transports Strasbourgeois (CTS)", "country": "FR", "feed_url": "https://opendata.cts-strasbourg.eu/google_transit.zip", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w\\d]+)", + "route_type_regex": "CTS", + "trip_number_regex": "", + "long_distance_regex": "^$", + "excluded_agencies": [] } }, { @@ -56,7 +86,13 @@ "name": "Eurostar", "country": "FR", "feed_url": "https://www.data.gouv.fr/fr/datasets/r/9089b550-696e-4ae0-87b5-40ea55a14292", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": false, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "Eurostar", + "trip_number_regex": "trip_short_name:(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -66,7 +102,13 @@ "name": "Trenitalia France", "country": "FR", "feed_url": "https://thello.axelor.com/public/gtfs/gtfs.zip", - "rt_feed_url": "https://thello.axelor.com/public/gtfs/GTFS-RT.bin" + "rt_feed_url": "https://thello.axelor.com/public/gtfs/GTFS-RT.bin", + "categorize_routes": false, + "route_name_regex": "route_short_name:([\\w\\s/-]+)", + "route_type_regex": "Trenitalia France", + "trip_number_regex": "trip_id:IT-FRA-TI-(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -76,7 +118,13 @@ "name": "Renfe", "country": "ES", "feed_url": "https://ssl.renfe.com/gtransit/Fichero_AV_LD/google_transit.zip", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "RENFE-&route_short_name:([\\w\\s]+)", + "trip_number_regex": "trip_short_name:(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -86,7 +134,13 @@ "name": "ÖBB", "country": "AT", "feed_url": "https://static.oebb.at/open-data/soll-fahrplan-gtfs/GTFS_OP_2024_obb.zip", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "ÖBB-&trip_short_name:(\\w+)\\s", + "trip_number_regex": "trip_short_name:\\w+\\s(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [] } }, { @@ -96,7 +150,15 @@ "name": "Transports suisses", "country": "CH", "feed_url": "https://opentransportdata.swiss/fr/dataset/timetable-2024-gtfs2020/permalink", - "rt_feed_url": "https://api.opentransportdata.swiss/gtfsrt2020" + "rt_feed_url": "https://api.opentransportdata.swiss/gtfsrt2020", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w\\s]*)", + "route_type_regex": "CH-&route_desc:([\\w\\s]*)", + "trip_number_regex": "trip_short_name:(\\d+)", + "long_distance_regex": "", + "excluded_agencies": [ + "CH-ALL-87_LEX" + ] } }, { @@ -106,7 +168,13 @@ "name": "CFL", "country": "LU", "feed_url": "https://data.public.lu/fr/datasets/r/aab2922d-27ff-4e53-a789-d990cf1ceb1e", - "rt_feed_url": "" + "rt_feed_url": "", + "categorize_routes": true, + "route_name_regex": "route_short_name:([\\w\\s]+)", + "route_type_regex": "CFL-&route_short_name:([\\w\\s]+)", + "trip_number_regex": "trip_short_name:(\\d+)", + "long_distance_regex": "route_type:3", + "excluded_agencies": [] } } ] diff --git a/trainvel/gtfs/management/commands/augment_data.py b/trainvel/gtfs/management/commands/augment_data.py new file mode 100644 index 0000000..0d920ef --- /dev/null +++ b/trainvel/gtfs/management/commands/augment_data.py @@ -0,0 +1,20 @@ +from django.core.management import BaseCommand +from django.db import transaction + +from tqdm import tqdm +from trainvel.gtfs.models import Trip + + +class Command(BaseCommand): + def add_arguments(self, parser): + parser.add_argument('--feed', type=str, help='GTFS Feed code') + + def handle(self, *args, **options): + trips = Trip.objects.filter(route_name="").prefetch_related('gtfs_feed', 'route') + if options['feed']: + trips = trips.filter(gtfs_feed__code=options['feed']) + + with transaction.atomic(): + for trip in tqdm(trips.all()): + trip.augment_data() + trip.save() diff --git a/trainvel/gtfs/migrations/0003_gtfsfeed_categorize_routes_and_more.py b/trainvel/gtfs/migrations/0003_gtfsfeed_categorize_routes_and_more.py new file mode 100644 index 0000000..542dcca --- /dev/null +++ b/trainvel/gtfs/migrations/0003_gtfsfeed_categorize_routes_and_more.py @@ -0,0 +1,72 @@ +# Generated by Django 5.0.6 on 2024-05-12 17:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("gtfs", "0002_alter_stop_parent_station"), + ] + + operations = [ + migrations.AddField( + model_name="gtfsfeed", + name="categorize_routes", + field=models.BooleanField( + default=False, + help_text="If checked, trips can be categorized by route type.", + verbose_name="categorize routes", + ), + ), + migrations.AddField( + model_name="gtfsfeed", + name="excluded_agencies", + field=models.ManyToManyField( + blank=True, + help_text="Agencies that are part of another feed and shouldn't be displayed with this feed.", + to="gtfs.agency", + verbose_name="excluded agencies", + ), + ), + migrations.AddField( + model_name="gtfsfeed", + name="long_distance_regex", + field=models.TextField( + blank=True, + default="", + help_text="Regular expression that filters long distance trips.", + verbose_name="long distance regex", + ), + ), + migrations.AddField( + model_name="gtfsfeed", + name="route_name_regex", + field=models.TextField( + blank=True, + default="route_short_name:([^§]+)", + help_text="Regular expression that catches the route name from a trip.", + verbose_name="route name regex", + ), + ), + migrations.AddField( + model_name="gtfsfeed", + name="route_type_regex", + field=models.TextField( + blank=True, + default="gtfs_feed:([^§]+)&-&route_type:([^§]+)", + help_text="Regular expression that catches the route type from a trip.", + verbose_name="route name regex", + ), + ), + migrations.AddField( + model_name="gtfsfeed", + name="trip_number_regex", + field=models.TextField( + blank=True, + default="trip_short_name:([^§]+)", + help_text="Regular expression that catches the trip number from a trip.", + verbose_name="route name regex", + ), + ), + ] diff --git a/trainvel/gtfs/migrations/0004_trip_long_distance_trip_route_name_trip_route_type_and_more.py b/trainvel/gtfs/migrations/0004_trip_long_distance_trip_route_name_trip_route_type_and_more.py new file mode 100644 index 0000000..7e7875b --- /dev/null +++ b/trainvel/gtfs/migrations/0004_trip_long_distance_trip_route_name_trip_route_type_and_more.py @@ -0,0 +1,39 @@ +# Generated by Django 5.0.6 on 2024-05-12 18:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("gtfs", "0003_gtfsfeed_categorize_routes_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="trip", + name="long_distance", + field=models.BooleanField(default=True, verbose_name="Long distance trip"), + ), + migrations.AddField( + model_name="trip", + name="route_name", + field=models.CharField( + blank=True, default="", max_length=255, verbose_name="Route name" + ), + ), + migrations.AddField( + model_name="trip", + name="route_type", + field=models.CharField( + blank=True, default="", max_length=255, verbose_name="Route type" + ), + ), + migrations.AddField( + model_name="trip", + name="trip_number", + field=models.CharField( + blank=True, default="", max_length=255, verbose_name="Trip number" + ), + ), + ] diff --git a/trainvel/gtfs/models.py b/trainvel/gtfs/models.py index afd465c..3e2254e 100644 --- a/trainvel/gtfs/models.py +++ b/trainvel/gtfs/models.py @@ -1,3 +1,5 @@ +import re + from django.db import models from django.utils.translation import gettext_lazy as _ @@ -56,6 +58,15 @@ class Country(models.TextChoices): UKRAINE = "UA", _("Ukraine") +class CategorizeTrips(models.TextChoices): + """ + Make trips be categorized by route type. + """ + GTFS_FEED = "gtfs_feed", _("GTFS Feed (all trips of the dataset)") + ROUTE_SHORT_NAME = "route_short_name", _("By route short name (e.g. A, S9,…)") + TRIP_SHORT_NAME_SPLIT = "route_short_name_split", _("By first character of the route short name (e.g. RJ, NJ,…)") + + class LocationType(models.IntegerChoices): STOP_PLATFORM = 0, _("Stop/platform") STATION = 1, _("Station") @@ -172,6 +183,47 @@ class GTFSFeed(models.Model): "If it is not modified, the file is the same."), ) + excluded_agencies = models.ManyToManyField( + to="Agency", + verbose_name=_("excluded agencies"), + blank=True, + help_text=_("Agencies that are part of another feed and shouldn't be displayed with this feed."), + ) + + categorize_routes = models.BooleanField( + verbose_name=_("categorize routes"), + default=False, + help_text=_("If checked, trips can be categorized by route type."), + ) + + route_name_regex = models.TextField( + verbose_name=_("route name regex"), + blank=True, + default=r"route_short_name:([^§]+)", + help_text=_("Regular expression that catches the route name from a trip."), + ) + + route_type_regex = models.TextField( + verbose_name=_("route name regex"), + blank=True, + default=r"gtfs_feed:([^§]+)&-&route_type:([^§]+)", + help_text=_("Regular expression that catches the route type from a trip."), + ) + + trip_number_regex = models.TextField( + verbose_name=_("route name regex"), + blank=True, + default=r"trip_short_name:([^§]+)", + help_text=_("Regular expression that catches the trip number from a trip."), + ) + + long_distance_regex = models.TextField( + verbose_name=_("long distance regex"), + blank=True, + default="", + help_text=_("Regular expression that filters long distance trips."), + ) + def __str__(self): return f"{self.name} ({self.code})" @@ -486,6 +538,36 @@ class Trip(models.Model): verbose_name=_("GTFS feed"), ) + """ + Augmented data + """ + + route_name = models.CharField( + max_length=255, + verbose_name=_("Route name"), + blank=True, + default="", + ) + + trip_number = models.CharField( + max_length=255, + verbose_name=_("Trip number"), + blank=True, + default="", + ) + + route_type = models.CharField( + max_length=255, + verbose_name=_("Route type"), + blank=True, + default="", + ) + + long_distance = models.BooleanField( + verbose_name=_("Long distance trip"), + default=True, + ) + @property def origin(self) -> Stop | None: return self.stop_times.order_by('stop_sequence').first().stop if self.stop_times.exists() else None @@ -556,6 +638,28 @@ class Trip(models.Model): origin_destination.fget.short_description = _("Origin → Destination") + def augment_data(self): + first_stop = self.stop_times.first() + if not first_stop: + return + desc_line = first_stop.description_line + + for key in ['route_name', 'route_type', 'trip_number']: + value = "" + full_pattern = getattr(self.gtfs_feed, f"{key}_regex") + for pattern in full_pattern.split('&'): + if '(' not in pattern: + value += pattern + continue + + groups = re.findall(pattern, desc_line) + if not any(group is not None for group in groups): + raise ValueError(f"Pattern {pattern} not found in {desc_line}") + value += "".join(group for group in groups if group) + setattr(self, key, value) + + self.long_distance = bool(re.match(self.gtfs_feed.long_distance_regex, desc_line)) + def __str__(self): return self.origin_destination @@ -638,6 +742,28 @@ class StopTime(models.Model): minutes = int((seconds % 3600) // 60) return f"{hours:02}:{minutes:02}" + @property + def description_line(self) -> str: + return (f"gtfs_feed:{self.trip.gtfs_feed_id}" + f"§stop:{self.stop_id}" + f"§trip_id:{self.trip_id}" + f"§trip_short_name:{self.trip.short_name}" + f"§trip_headsign:{self.trip.headsign}" + f"§route_short_name:{self.trip.route.short_name}" + f"§route_desc:{self.trip.route.desc}" + f"§route_type:{self.trip.route.type}") + + @property + def description_line_expr(self): + return "gtfs_feed:" + models.F('trip__gtfs_feed_id') \ + + "§stop:" + models.F('stop_id') \ + + "§trip_id:" + models.F('trip_id') \ + + "§trip_short_name:" + models.F('trip__short_name') \ + + "§trip_headsign:" + models.F('trip__headsign') \ + + "§route_short_name:" + models.F('trip__route__short_name') \ + + "§route_desc:" + models.F('trip__route__desc') \ + + "§route_type:" + models.F('trip__route__type') + def __str__(self): return f"{self.stop.name} - {self.trip_id}" @@ -760,6 +886,46 @@ class Transfer(models.Model): related_name="transfers_to", ) + from_route = models.ForeignKey( + to="Route", + on_delete=models.CASCADE, + verbose_name=_("From route"), + related_name="transfers_from", + null=True, + blank=True, + default=None, + ) + + to_route = models.ForeignKey( + to="Route", + on_delete=models.CASCADE, + verbose_name=_("To route"), + related_name="transfers_to", + null=True, + blank=True, + default=None, + ) + + from_trip = models.ForeignKey( + to="Trip", + on_delete=models.CASCADE, + verbose_name=_("From trip"), + related_name="transfers_from", + null=True, + blank=True, + default=None, + ) + + to_trip = models.ForeignKey( + to="Trip", + on_delete=models.CASCADE, + verbose_name=_("To trip"), + related_name="transfers_to", + null=True, + blank=True, + default=None, + ) + transfer_type = models.IntegerField( verbose_name=_("Transfer type"), choices=TransferType, @@ -769,6 +935,7 @@ class Transfer(models.Model): min_transfer_time = models.IntegerField( verbose_name=_("Minimum transfer time"), blank=True, + default=None, ) class Meta: