From 77c3ef9e74ef46400d4b54d6ec903357fd035148 Mon Sep 17 00:00:00 2001 From: Emmy D'Anello Date: Fri, 9 Feb 2024 23:15:14 +0100 Subject: [PATCH] Truncate trip id --- sncfgtfs/admin.py | 40 +++++- .../management/commands/update_sncf_gtfs.py | 68 +++++++--- sncfgtfs/migrations/0001_initial.py | 126 +++++++++++++++--- ..._trip_options_tripupdate_stoptimeupdate.py | 108 --------------- sncfgtfs/models.py | 42 ++++-- 5 files changed, 225 insertions(+), 159 deletions(-) delete mode 100644 sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py diff --git a/sncfgtfs/admin.py b/sncfgtfs/admin.py index 6faad63..2345f0c 100644 --- a/sncfgtfs/admin.py +++ b/sncfgtfs/admin.py @@ -4,6 +4,37 @@ from sncfgtfs.models import Agency, Stop, Route, Trip, StopTime, Calendar, Calen Transfer, FeedInfo, StopTimeUpdate, TripUpdate +class CalendarDateInline(admin.TabularInline): + model = CalendarDate + extra = 0 + + +class TripInline(admin.TabularInline): + model = Trip + extra = 0 + autocomplete_fields = ('route', 'service',) + show_change_link = True + ordering = ('service',) + + +class StopTimeInline(admin.TabularInline): + model = StopTime + extra = 0 + autocomplete_fields = ('stop',) + show_change_link = True + ordering = ('stop_sequence',) + + +class TripUpdateInline(admin.StackedInline): + model = TripUpdate + extra = 0 + + +class StopTimeUpdateInline(admin.StackedInline): + model = StopTimeUpdate + extra = 0 + + @admin.register(Agency) class AgencyAdmin(admin.ModelAdmin): list_display = ('name', 'id', 'url', 'timezone',) @@ -26,24 +57,28 @@ class RouteAdmin(admin.ModelAdmin): search_fields = ('long_name', 'short_name', 'id',) ordering = ('long_name',) autocomplete_fields = ('agency',) + inlines = (TripInline,) @admin.register(Trip) class TripAdmin(admin.ModelAdmin): list_display = ('id', 'route', 'service', 'headsign', 'direction_id',) - list_filter = ('direction_id', 'service__transport_type',) + list_filter = ('direction_id', 'route__transport_type',) search_fields = ('id', 'route__id', 'route__long_name', 'service__id', 'headsign',) ordering = ('route', 'service',) + autocomplete_fields = ('route', 'service',) + inlines = (StopTimeInline, TripUpdateInline,) @admin.register(StopTime) class StopTimeAdmin(admin.ModelAdmin): list_display = ('trip', 'stop', 'arrival_time', 'departure_time', 'stop_sequence', 'pickup_type', 'drop_off_type',) - list_filter = ('pickup_type', 'drop_off_type', 'trip__service__transport_type',) + list_filter = ('pickup_type', 'drop_off_type', 'trip__route__transport_type',) search_fields = ('trip__id', 'stop__name', 'arrival_time', 'departure_time',) ordering = ('trip', 'stop_sequence',) autocomplete_fields = ('trip', 'stop',) + inlines = (StopTimeUpdateInline,) @admin.register(Calendar) @@ -54,6 +89,7 @@ class CalendarAdmin(admin.ModelAdmin): 'start_date', 'end_date',) search_fields = ('id', 'start_date', 'end_date',) ordering = ('transport_type', 'id',) + inlines = (CalendarDateInline, TripInline,) @admin.register(CalendarDate) diff --git a/sncfgtfs/management/commands/update_sncf_gtfs.py b/sncfgtfs/management/commands/update_sncf_gtfs.py index 2d54ece..5972216 100644 --- a/sncfgtfs/management/commands/update_sncf_gtfs.py +++ b/sncfgtfs/management/commands/update_sncf_gtfs.py @@ -20,10 +20,17 @@ class Command(BaseCommand): } def add_arguments(self, parser): - parser.add_argument('--bulk_size', type=int, default=1000, help='Number of objects to create in bulk.') + parser.add_argument('--bulk_size', type=int, default=1000, help="Number of objects to create in bulk.") + parser.add_argument('--dry-run', action='store_true', + help="Do not update the database, only print what would be done.") + parser.add_argument('--force', '-f', action='store_true', help="Force the update of the database.") def handle(self, *args, **options): bulk_size = options['bulk_size'] + dry_run = options['dry_run'] + force = options['force'] + if dry_run: + self.stdout.write(self.style.WARNING("Dry run mode activated.")) if not FeedInfo.objects.exists(): last_update_date = "1970-01-01" @@ -36,11 +43,14 @@ class Command(BaseCommand): if last_modified.date().isoformat() > last_update_date: break else: - self.stdout.write(self.style.WARNING("Database already up-to-date.")) - return + if not force: + self.stdout.write(self.style.WARNING("Database already up-to-date.")) + return self.stdout.write("Updating database...") + all_trips = [] + for transport_type, feed_url in self.GTFS_FEEDS.items(): self.stdout.write(f"Downloading {transport_type} GTFS feed...") with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile: @@ -57,7 +67,7 @@ class Command(BaseCommand): email=agency_dict.get('agency_email', ""), ) agencies.append(agency) - if agencies: + if agencies and not dry_run: Agency.objects.bulk_create(agencies, update_conflicts=True, update_fields=['name', 'url', 'timezone', 'lang', 'phone', 'email'], @@ -85,7 +95,7 @@ class Command(BaseCommand): ) stops.append(stop) - if len(stops) >= bulk_size: + if len(stops) >= bulk_size and not dry_run: Stop.objects.bulk_create(stops, update_conflicts=True, update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url', @@ -93,7 +103,7 @@ class Command(BaseCommand): 'wheelchair_boarding', 'level_id', 'platform_code'], unique_fields=['id']) stops.clear() - if stops: + if stops and not dry_run: Stop.objects.bulk_create(stops, update_conflicts=True, update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url', @@ -115,17 +125,19 @@ class Command(BaseCommand): url=route_dict['route_url'], color=route_dict['route_color'], text_color=route_dict['route_text_color'], + transport_type=transport_type, ) routes.append(route) - if len(routes) >= bulk_size: + if len(routes) >= bulk_size and not dry_run: Route.objects.bulk_create(routes, update_conflicts=True, update_fields=['agency_id', 'short_name', 'long_name', 'desc', - 'type', 'url', 'color', 'text_color'], + 'type', 'url', 'color', 'text_color', + 'transport_type'], unique_fields=['id']) routes.clear() - if routes: + if routes and not dry_run: Route.objects.bulk_create(routes, update_conflicts=True, update_fields=['agency_id', 'short_name', 'long_name', 'desc', @@ -154,7 +166,7 @@ class Command(BaseCommand): calendars.append(calendar) calendar_ids.append(calendar.id) - if len(calendars) >= bulk_size: + if len(calendars) >= bulk_size and not dry_run: Calendar.objects.bulk_create(calendars, update_conflicts=True, update_fields=['monday', 'tuesday', 'wednesday', 'thursday', @@ -162,7 +174,7 @@ class Command(BaseCommand): 'end_date', 'transport_type'], unique_fields=['id']) calendars.clear() - if calendars: + if calendars and not dry_run: Calendar.objects.bulk_create(calendars, update_conflicts=True, update_fields=['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday', 'start_date', @@ -198,7 +210,7 @@ class Command(BaseCommand): ) calendars.append(calendar) - if len(calendar_dates) >= bulk_size: + if len(calendar_dates) >= bulk_size and not dry_run: Calendar.objects.bulk_create(calendars, update_conflicts=True, update_fields=['end_date'], @@ -210,7 +222,7 @@ class Command(BaseCommand): calendars.clear() calendar_dates.clear() - if calendar_dates: + if calendar_dates and not dry_run: Calendar.objects.bulk_create(calendars, update_conflicts=True, update_fields=['end_date'], @@ -225,8 +237,14 @@ class Command(BaseCommand): trips = [] for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()): trip_dict: dict + trip_id = trip_dict['trip_id'] + if transport_type != "TN": + trip_id, last_update = trip_id.split(':', 1) + last_update = datetime.fromisoformat(last_update) + else: + last_update = None trip = Trip( - id=trip_dict['trip_id'], + id=trip_id, route_id=trip_dict['route_id'], service_id=f"{transport_type}-{trip_dict['service_id']}", headsign=trip_dict['trip_headsign'], @@ -236,10 +254,11 @@ class Command(BaseCommand): shape_id=trip_dict['shape_id'], wheelchair_accessible=trip_dict.get('wheelchair_accessible', None), bikes_allowed=trip_dict.get('bikes_allowed', None), + last_update=last_update, ) trips.append(trip) - if len(trips) >= bulk_size: + if len(trips) >= bulk_size and not dry_run: Trip.objects.bulk_create(trips, update_conflicts=True, update_fields=['route_id', 'service_id', 'headsign', 'short_name', @@ -247,7 +266,7 @@ class Command(BaseCommand): 'wheelchair_accessible', 'bikes_allowed'], unique_fields=['id']) trips.clear() - if trips: + if trips and not dry_run: Trip.objects.bulk_create(trips, update_conflicts=True, update_fields=['route_id', 'service_id', 'headsign', 'short_name', @@ -256,17 +275,22 @@ class Command(BaseCommand): unique_fields=['id']) trips.clear() + all_trips.extend(trips) + stop_times = [] for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()): stop_time_dict: dict + trip_id = stop_time_dict['trip_id'] + if transport_type != "TN": + trip_id = trip_id.split(':', 1)[0] arr_time = stop_time_dict['arrival_time'] arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:]) dep_time = stop_time_dict['departure_time'] dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:]) st = StopTime( id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}", - trip_id=stop_time_dict['trip_id'], + trip_id=trip_id, arrival_time=timedelta(seconds=arr_time), departure_time=timedelta(seconds=dep_time), stop_id=stop_time_dict['stop_id'], @@ -278,7 +302,7 @@ class Command(BaseCommand): ) stop_times.append(st) - if len(stop_times) >= bulk_size: + if len(stop_times) >= bulk_size and not dry_run: StopTime.objects.bulk_create(stop_times, update_conflicts=True, update_fields=['stop_id', 'arrival_time', 'departure_time', @@ -286,7 +310,7 @@ class Command(BaseCommand): 'drop_off_type', 'timepoint'], unique_fields=['id']) stop_times.clear() - if stop_times: + if stop_times and not dry_run: StopTime.objects.bulk_create(stop_times, update_conflicts=True, update_fields=['stop_id', 'arrival_time', 'departure_time', @@ -307,21 +331,21 @@ class Command(BaseCommand): ) transfers.append(transfer) - if len(transfers) >= bulk_size: + if len(transfers) >= bulk_size and not dry_run: Transfer.objects.bulk_create(transfers, update_conflicts=True, update_fields=['transfer_type', 'min_transfer_time'], unique_fields=['id']) transfers.clear() - if transfers: + if transfers and not dry_run: Transfer.objects.bulk_create(transfers, update_conflicts=True, update_fields=['transfer_type', 'min_transfer_time'], unique_fields=['id']) transfers.clear() - if "feed_info.txt" in zipfile.namelist(): + if "feed_info.txt" in zipfile.namelist() and not dry_run: for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()): feed_info_dict: dict FeedInfo.objects.update_or_create( diff --git a/sncfgtfs/migrations/0001_initial.py b/sncfgtfs/migrations/0001_initial.py index 8c14914..9fd72bc 100644 --- a/sncfgtfs/migrations/0001_initial.py +++ b/sncfgtfs/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 5.0.1 on 2024-01-27 14:08 +# Generated by Django 5.0.1 on 2024-02-09 21:55 import django.db.models.deletion from django.db import migrations, models @@ -155,19 +155,6 @@ class Migration(migrations.Migration): verbose_name="Exception type", ), ), - ( - "transport_type", - models.CharField( - choices=[ - ("TGV", "TGV"), - ("TER", "TER"), - ("IC", "Intercités"), - ("TN", "Transilien"), - ], - max_length=255, - verbose_name="Transport type", - ), - ), ( "service", models.ForeignKey( @@ -239,13 +226,26 @@ class Migration(migrations.Migration): blank=True, max_length=255, verbose_name="Route text color" ), ), + ( + "transport_type", + models.CharField( + choices=[ + ("TGV", "TGV"), + ("TER", "TER"), + ("IC", "Intercités"), + ("TN", "Transilien"), + ], + max_length=255, + verbose_name="Transport type", + ), + ), ( "agency", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, related_name="routes", to="sncfgtfs.agency", - verbose_name="Agency ID", + verbose_name="Agency", ), ), ], @@ -474,6 +474,10 @@ class Migration(migrations.Migration): verbose_name="Bikes allowed", ), ), + ( + "last_update", + models.DateTimeField(null=True, verbose_name="Last update"), + ), ( "route", models.ForeignKey( @@ -496,7 +500,6 @@ class Migration(migrations.Migration): options={ "verbose_name": "Trip", "verbose_name_plural": "Trips", - "ordering": ("id",), }, ), migrations.CreateModel( @@ -578,4 +581,95 @@ class Migration(migrations.Migration): "verbose_name_plural": "Stop times", }, ), + migrations.CreateModel( + name="TripUpdate", + fields=[ + ( + "trip", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="update", + serialize=False, + to="sncfgtfs.trip", + verbose_name="Trip", + ), + ), + ("start_date", models.DateField(verbose_name="Start date")), + ("start_time", models.TimeField(verbose_name="Start time")), + ( + "schedule_relationship", + models.IntegerField( + choices=[ + (0, "Scheduled"), + (1, "Added"), + (2, "Unscheduled"), + (3, "Canceled"), + (5, "Replacement"), + (6, "Duplicated"), + (7, "Deleted"), + ], + default=0, + verbose_name="Schedule relationship", + ), + ), + ], + options={ + "verbose_name": "Trip update", + "verbose_name_plural": "Trip updates", + "ordering": ("start_date", "trip"), + "unique_together": {("trip", "start_date", "start_time")}, + }, + ), + migrations.CreateModel( + name="StopTimeUpdate", + fields=[ + ( + "stop_time", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="update", + serialize=False, + to="sncfgtfs.stoptime", + verbose_name="Stop time", + ), + ), + ("arrival_delay", models.DurationField(verbose_name="Arrival delay")), + ("arrival_time", models.DateTimeField(verbose_name="Arrival time")), + ( + "departure_delay", + models.DurationField(verbose_name="Departure delay"), + ), + ("departure_time", models.DateTimeField(verbose_name="Departure time")), + ( + "schedule_relationship", + models.IntegerField( + choices=[ + (0, "Scheduled"), + (1, "Skipped"), + (2, "No data"), + (3, "Unscheduled"), + ], + default=0, + verbose_name="Schedule relationship", + ), + ), + ( + "trip_update", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="stop_time_updates", + to="sncfgtfs.tripupdate", + verbose_name="Trip update", + ), + ), + ], + options={ + "verbose_name": "Stop time update", + "verbose_name_plural": "Stop time updates", + "ordering": ("trip_update", "stop_time"), + "unique_together": {("trip_update", "stop_time")}, + }, + ), ] diff --git a/sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py b/sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py deleted file mode 100644 index e770d5c..0000000 --- a/sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py +++ /dev/null @@ -1,108 +0,0 @@ -# Generated by Django 5.0.1 on 2024-02-06 06:59 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("sncfgtfs", "0001_initial"), - ] - - operations = [ - migrations.AlterModelOptions( - name="trip", - options={"verbose_name": "Trip", "verbose_name_plural": "Trips"}, - ), - migrations.CreateModel( - name="TripUpdate", - fields=[ - ( - "trip", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="update", - serialize=False, - to="sncfgtfs.trip", - verbose_name="Trip", - ), - ), - ("start_date", models.DateField(verbose_name="Start date")), - ("start_time", models.TimeField(verbose_name="Start time")), - ( - "schedule_relationship", - models.IntegerField( - choices=[ - (0, "Scheduled"), - (1, "Added"), - (2, "Unscheduled"), - (3, "Canceled"), - (5, "Replacement"), - (6, "Duplicated"), - (7, "Deleted"), - ], - default=0, - verbose_name="Schedule relationship", - ), - ), - ], - options={ - "verbose_name": "Trip update", - "verbose_name_plural": "Trip updates", - "ordering": ("start_date", "trip"), - "unique_together": {("trip", "start_date", "start_time")}, - }, - ), - migrations.CreateModel( - name="StopTimeUpdate", - fields=[ - ( - "stop_time", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="update", - serialize=False, - to="sncfgtfs.stoptime", - verbose_name="Stop time", - ), - ), - ("arrival_delay", models.DurationField(verbose_name="Arrival delay")), - ("arrival_time", models.DateTimeField(verbose_name="Arrival time")), - ( - "departure_delay", - models.DurationField(verbose_name="Departure delay"), - ), - ("departure_time", models.DateTimeField(verbose_name="Departure time")), - ( - "schedule_relationship", - models.IntegerField( - choices=[ - (0, "Scheduled"), - (1, "Skipped"), - (2, "No data"), - (3, "Unscheduled"), - ], - default=0, - verbose_name="Schedule relationship", - ), - ), - ( - "trip_update", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="stop_time_updates", - to="sncfgtfs.tripupdate", - verbose_name="Trip update", - ), - ), - ], - options={ - "verbose_name": "Stop time update", - "verbose_name_plural": "Stop time updates", - "ordering": ("trip_update", "stop_time"), - "unique_together": {("trip_update", "stop_time")}, - }, - ), - ] diff --git a/sncfgtfs/models.py b/sncfgtfs/models.py index 36fe593..0410581 100644 --- a/sncfgtfs/models.py +++ b/sncfgtfs/models.py @@ -230,7 +230,7 @@ class Route(models.Model): agency = models.ForeignKey( to="Agency", on_delete=models.CASCADE, - verbose_name=_("Agency ID"), + verbose_name=_("Agency"), related_name="routes", ) @@ -272,6 +272,12 @@ class Route(models.Model): blank=True, ) + transport_type = models.CharField( + max_length=255, + verbose_name=_("Transport type"), + choices=TransportType, + ) + def __str__(self): return f"{self.long_name}" @@ -346,6 +352,11 @@ class Trip(models.Model): null=True, ) + last_update = models.DateTimeField( + verbose_name=_("Last update"), + null=True, + ) + @property def origin(self): return self.stop_times.order_by('stop_sequence').first().stop @@ -354,16 +365,30 @@ class Trip(models.Model): def destination(self): return self.stop_times.order_by('-stop_sequence').first().stop + @property + def departure_time(self): + dep_time = self.stop_times.order_by('stop_sequence').first().departure_time + hours = int(dep_time.total_seconds() // 3600) + minutes = int((dep_time.total_seconds() % 3600) // 60) + return f"{hours:02}:{minutes:02}" + + @property + def arrival_time(self): + arr_time = self.stop_times.order_by('-stop_sequence').first().arrival_time + hours = int(arr_time.total_seconds() // 3600) + minutes = int((arr_time.total_seconds() % 3600) // 60) + return f"{hours:02}:{minutes:02}" + @property def train_type(self): - if self.service.transport_type == TransportType.TRANSILIEN: + if self.route.transport_type == TransportType.TRANSILIEN: return self.route.short_name else: return self.origin.stop_type @property def train_number(self): - if self.service.transport_type == TransportType.TRANSILIEN: + if self.route.transport_type == TransportType.TRANSILIEN: return self.short_name else: return self.headsign @@ -389,7 +414,8 @@ class Trip(models.Model): return "000000" def __str__(self): - return f"{self.route.long_name} - {self.id}" + return f"{self.origin.name} {self.departure_time} → {self.destination.name} {self.arrival_time}" \ + f" - {self.service_id}" class Meta: verbose_name = _("Trip") @@ -470,7 +496,7 @@ class StopTime(models.Model): return f"{hours:02}:{minutes:02}" def __str__(self): - return f"{self.trip.route.long_name} - {self.trip_id} - {self.stop.name}" + return f"{self.stop.name} - {self.trip_id}" class Meta: verbose_name = _("Stop time") @@ -558,12 +584,6 @@ class CalendarDate(models.Model): choices=ExceptionType, ) - transport_type = models.CharField( - max_length=255, - verbose_name=_("Transport type"), - choices=TransportType, - ) - def __str__(self): return f"{self.service.id} - {self.date} - {self.exception_type}"