Truncate trip id

sncfgtfs/admin.py

@@ -4,6 +4,37 @@ from sncfgtfs.models import Agency, Stop, Route, Trip, StopTime, Calendar, Calen
     Transfer, FeedInfo, StopTimeUpdate, TripUpdate
 
 
+class CalendarDateInline(admin.TabularInline):
+    model = CalendarDate
+    extra = 0
+
+
+class TripInline(admin.TabularInline):
+    model = Trip
+    extra = 0
+    autocomplete_fields = ('route', 'service',)
+    show_change_link = True
+    ordering = ('service',)
+
+
+class StopTimeInline(admin.TabularInline):
+    model = StopTime
+    extra = 0
+    autocomplete_fields = ('stop',)
+    show_change_link = True
+    ordering = ('stop_sequence',)
+
+
+class TripUpdateInline(admin.StackedInline):
+    model = TripUpdate
+    extra = 0
+
+
+class StopTimeUpdateInline(admin.StackedInline):
+    model = StopTimeUpdate
+    extra = 0
+
+
 @admin.register(Agency)
 class AgencyAdmin(admin.ModelAdmin):
     list_display = ('name', 'id', 'url', 'timezone',)
@@ -26,24 +57,28 @@ class RouteAdmin(admin.ModelAdmin):
     search_fields = ('long_name', 'short_name', 'id',)
     ordering = ('long_name',)
     autocomplete_fields = ('agency',)
+    inlines = (TripInline,)
 
 
 @admin.register(Trip)
 class TripAdmin(admin.ModelAdmin):
     list_display = ('id', 'route', 'service', 'headsign', 'direction_id',)
-    list_filter = ('direction_id', 'service__transport_type',)
+    list_filter = ('direction_id', 'route__transport_type',)
     search_fields = ('id', 'route__id', 'route__long_name', 'service__id', 'headsign',)
     ordering = ('route', 'service',)
+    autocomplete_fields = ('route', 'service',)
+    inlines = (StopTimeInline, TripUpdateInline,)
 
 
 @admin.register(StopTime)
 class StopTimeAdmin(admin.ModelAdmin):
     list_display = ('trip', 'stop', 'arrival_time', 'departure_time',
                     'stop_sequence', 'pickup_type', 'drop_off_type',)
-    list_filter = ('pickup_type', 'drop_off_type', 'trip__service__transport_type',)
+    list_filter = ('pickup_type', 'drop_off_type', 'trip__route__transport_type',)
     search_fields = ('trip__id', 'stop__name', 'arrival_time', 'departure_time',)
     ordering = ('trip', 'stop_sequence',)
     autocomplete_fields = ('trip', 'stop',)
+    inlines = (StopTimeUpdateInline,)
 
 
 @admin.register(Calendar)
@@ -54,6 +89,7 @@ class CalendarAdmin(admin.ModelAdmin):
                    'start_date', 'end_date',)
     search_fields = ('id', 'start_date', 'end_date',)
     ordering = ('transport_type', 'id',)
+    inlines = (CalendarDateInline, TripInline,)
 
 
 @admin.register(CalendarDate)

sncfgtfs/management/commands/update_sncf_gtfs.py

@@ -20,10 +20,17 @@ class Command(BaseCommand):
     }
 
     def add_arguments(self, parser):
-        parser.add_argument('--bulk_size', type=int, default=1000, help='Number of objects to create in bulk.')
+        parser.add_argument('--bulk_size', type=int, default=1000, help="Number of objects to create in bulk.")
+        parser.add_argument('--dry-run', action='store_true',
+                            help="Do not update the database, only print what would be done.")
+        parser.add_argument('--force', '-f', action='store_true', help="Force the update of the database.")
 
     def handle(self, *args, **options):
         bulk_size = options['bulk_size']
+        dry_run = options['dry_run']
+        force = options['force']
+        if dry_run:
+            self.stdout.write(self.style.WARNING("Dry run mode activated."))
 
         if not FeedInfo.objects.exists():
             last_update_date = "1970-01-01"
@@ -36,11 +43,14 @@ class Command(BaseCommand):
             if last_modified.date().isoformat() > last_update_date:
                 break
         else:
-            self.stdout.write(self.style.WARNING("Database already up-to-date."))
+            if not force:
-            return
+                self.stdout.write(self.style.WARNING("Database already up-to-date."))
+                return
 
         self.stdout.write("Updating database...")
 
+        all_trips = []
+
         for transport_type, feed_url in self.GTFS_FEEDS.items():
             self.stdout.write(f"Downloading {transport_type} GTFS feed...")
             with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile:
@@ -57,7 +67,7 @@ class Command(BaseCommand):
                         email=agency_dict.get('agency_email', ""),
                     )
                     agencies.append(agency)
-                if agencies:
+                if agencies and not dry_run:
                     Agency.objects.bulk_create(agencies,
                                                update_conflicts=True,
                                                update_fields=['name', 'url', 'timezone', 'lang', 'phone', 'email'],
@@ -85,7 +95,7 @@ class Command(BaseCommand):
                     )
                     stops.append(stop)
 
-                    if len(stops) >= bulk_size:
+                    if len(stops) >= bulk_size and not dry_run:
                         Stop.objects.bulk_create(stops,
                                                  update_conflicts=True,
                                                  update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
@@ -93,7 +103,7 @@ class Command(BaseCommand):
                                                                 'wheelchair_boarding', 'level_id', 'platform_code'],
                                                  unique_fields=['id'])
                         stops.clear()
-                if stops:
+                if stops and not dry_run:
                     Stop.objects.bulk_create(stops,
                                              update_conflicts=True,
                                              update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
@@ -115,17 +125,19 @@ class Command(BaseCommand):
                         url=route_dict['route_url'],
                         color=route_dict['route_color'],
                         text_color=route_dict['route_text_color'],
+                        transport_type=transport_type,
                     )
                     routes.append(route)
 
-                    if len(routes) >= bulk_size:
+                    if len(routes) >= bulk_size and not dry_run:
                         Route.objects.bulk_create(routes,
                                                   update_conflicts=True,
                                                   update_fields=['agency_id', 'short_name', 'long_name', 'desc',
-                                                                 'type', 'url', 'color', 'text_color'],
+                                                                 'type', 'url', 'color', 'text_color',
+                                                                 'transport_type'],
                                                   unique_fields=['id'])
                         routes.clear()
-                if routes:
+                if routes and not dry_run:
                     Route.objects.bulk_create(routes,
                                               update_conflicts=True,
                                               update_fields=['agency_id', 'short_name', 'long_name', 'desc',
@@ -154,7 +166,7 @@ class Command(BaseCommand):
                         calendars.append(calendar)
                         calendar_ids.append(calendar.id)
 
-                        if len(calendars) >= bulk_size:
+                        if len(calendars) >= bulk_size and not dry_run:
                             Calendar.objects.bulk_create(calendars,
                                                          update_conflicts=True,
                                                          update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
@@ -162,7 +174,7 @@ class Command(BaseCommand):
                                                                         'end_date', 'transport_type'],
                                                          unique_fields=['id'])
                             calendars.clear()
-                    if calendars:
+                    if calendars and not dry_run:
                         Calendar.objects.bulk_create(calendars, update_conflicts=True,
                                                      update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
                                                                     'friday', 'saturday', 'sunday', 'start_date',
@@ -198,7 +210,7 @@ class Command(BaseCommand):
                         )
                         calendars.append(calendar)
 
-                    if len(calendar_dates) >= bulk_size:
+                    if len(calendar_dates) >= bulk_size and not dry_run:
                         Calendar.objects.bulk_create(calendars,
                                                      update_conflicts=True,
                                                      update_fields=['end_date'],
@@ -210,7 +222,7 @@ class Command(BaseCommand):
                         calendars.clear()
                         calendar_dates.clear()
 
-                if calendar_dates:
+                if calendar_dates and not dry_run:
                     Calendar.objects.bulk_create(calendars,
                                                  update_conflicts=True,
                                                  update_fields=['end_date'],
@@ -225,8 +237,14 @@ class Command(BaseCommand):
                 trips = []
                 for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()):
                     trip_dict: dict
+                    trip_id = trip_dict['trip_id']
+                    if transport_type != "TN":
+                        trip_id, last_update = trip_id.split(':', 1)
+                        last_update = datetime.fromisoformat(last_update)
+                    else:
+                        last_update = None
                     trip = Trip(
-                        id=trip_dict['trip_id'],
+                        id=trip_id,
                         route_id=trip_dict['route_id'],
                         service_id=f"{transport_type}-{trip_dict['service_id']}",
                         headsign=trip_dict['trip_headsign'],
@@ -236,10 +254,11 @@ class Command(BaseCommand):
                         shape_id=trip_dict['shape_id'],
                         wheelchair_accessible=trip_dict.get('wheelchair_accessible', None),
                         bikes_allowed=trip_dict.get('bikes_allowed', None),
+                        last_update=last_update,
                     )
                     trips.append(trip)
 
-                    if len(trips) >= bulk_size:
+                    if len(trips) >= bulk_size and not dry_run:
                         Trip.objects.bulk_create(trips,
                                                  update_conflicts=True,
                                                  update_fields=['route_id', 'service_id', 'headsign', 'short_name',
@@ -247,7 +266,7 @@ class Command(BaseCommand):
                                                                 'wheelchair_accessible', 'bikes_allowed'],
                                                  unique_fields=['id'])
                         trips.clear()
-                if trips:
+                if trips and not dry_run:
                     Trip.objects.bulk_create(trips,
                                              update_conflicts=True,
                                              update_fields=['route_id', 'service_id', 'headsign', 'short_name',
@@ -256,17 +275,22 @@ class Command(BaseCommand):
                                              unique_fields=['id'])
                     trips.clear()
 
+                all_trips.extend(trips)
+
                 stop_times = []
                 for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()):
                     stop_time_dict: dict
 
+                    trip_id = stop_time_dict['trip_id']
+                    if transport_type != "TN":
+                        trip_id = trip_id.split(':', 1)[0]
                     arr_time = stop_time_dict['arrival_time']
                     arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:])
                     dep_time = stop_time_dict['departure_time']
                     dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:])
                     st = StopTime(
                         id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}",
-                        trip_id=stop_time_dict['trip_id'],
+                        trip_id=trip_id,
                         arrival_time=timedelta(seconds=arr_time),
                         departure_time=timedelta(seconds=dep_time),
                         stop_id=stop_time_dict['stop_id'],
@@ -278,7 +302,7 @@ class Command(BaseCommand):
                     )
                     stop_times.append(st)
 
-                    if len(stop_times) >= bulk_size:
+                    if len(stop_times) >= bulk_size and not dry_run:
                         StopTime.objects.bulk_create(stop_times,
                                                      update_conflicts=True,
                                                      update_fields=['stop_id', 'arrival_time', 'departure_time',
@@ -286,7 +310,7 @@ class Command(BaseCommand):
                                                                     'drop_off_type', 'timepoint'],
                                                      unique_fields=['id'])
                         stop_times.clear()
-                if stop_times:
+                if stop_times and not dry_run:
                     StopTime.objects.bulk_create(stop_times,
                                                  update_conflicts=True,
                                                  update_fields=['stop_id', 'arrival_time', 'departure_time',
@@ -307,21 +331,21 @@ class Command(BaseCommand):
                     )
                     transfers.append(transfer)
 
-                    if len(transfers) >= bulk_size:
+                    if len(transfers) >= bulk_size and not dry_run:
                         Transfer.objects.bulk_create(transfers,
                                                      update_conflicts=True,
                                                      update_fields=['transfer_type', 'min_transfer_time'],
                                                      unique_fields=['id'])
                         transfers.clear()
 
-                if transfers:
+                if transfers and not dry_run:
                     Transfer.objects.bulk_create(transfers,
                                                  update_conflicts=True,
                                                  update_fields=['transfer_type', 'min_transfer_time'],
                                                  unique_fields=['id'])
                     transfers.clear()
 
-                if "feed_info.txt" in zipfile.namelist():
+                if "feed_info.txt" in zipfile.namelist() and not dry_run:
                     for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()):
                         feed_info_dict: dict
                         FeedInfo.objects.update_or_create(

sncfgtfs/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.1 on 2024-01-27 14:08
+# Generated by Django 5.0.1 on 2024-02-09 21:55
 
 import django.db.models.deletion
 from django.db import migrations, models
@@ -155,19 +155,6 @@ class Migration(migrations.Migration):
                         verbose_name="Exception type",
                     ),
                 ),
-                (
-                    "transport_type",
-                    models.CharField(
-                        choices=[
-                            ("TGV", "TGV"),
-                            ("TER", "TER"),
-                            ("IC", "Intercités"),
-                            ("TN", "Transilien"),
-                        ],
-                        max_length=255,
-                        verbose_name="Transport type",
-                    ),
-                ),
                 (
                     "service",
                     models.ForeignKey(
@@ -239,13 +226,26 @@ class Migration(migrations.Migration):
                         blank=True, max_length=255, verbose_name="Route text color"
                     ),
                 ),
+                (
+                    "transport_type",
+                    models.CharField(
+                        choices=[
+                            ("TGV", "TGV"),
+                            ("TER", "TER"),
+                            ("IC", "Intercités"),
+                            ("TN", "Transilien"),
+                        ],
+                        max_length=255,
+                        verbose_name="Transport type",
+                    ),
+                ),
                 (
                     "agency",
                     models.ForeignKey(
                         on_delete=django.db.models.deletion.CASCADE,
                         related_name="routes",
                         to="sncfgtfs.agency",
-                        verbose_name="Agency ID",
+                        verbose_name="Agency",
                     ),
                 ),
             ],
@@ -474,6 +474,10 @@ class Migration(migrations.Migration):
                         verbose_name="Bikes allowed",
                     ),
                 ),
+                (
+                    "last_update",
+                    models.DateTimeField(null=True, verbose_name="Last update"),
+                ),
                 (
                     "route",
                     models.ForeignKey(
@@ -496,7 +500,6 @@ class Migration(migrations.Migration):
             options={
                 "verbose_name": "Trip",
                 "verbose_name_plural": "Trips",
-                "ordering": ("id",),
             },
         ),
         migrations.CreateModel(
@@ -578,4 +581,95 @@ class Migration(migrations.Migration):
                 "verbose_name_plural": "Stop times",
             },
         ),
+        migrations.CreateModel(
+            name="TripUpdate",
+            fields=[
+                (
+                    "trip",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        primary_key=True,
+                        related_name="update",
+                        serialize=False,
+                        to="sncfgtfs.trip",
+                        verbose_name="Trip",
+                    ),
+                ),
+                ("start_date", models.DateField(verbose_name="Start date")),
+                ("start_time", models.TimeField(verbose_name="Start time")),
+                (
+                    "schedule_relationship",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Scheduled"),
+                            (1, "Added"),
+                            (2, "Unscheduled"),
+                            (3, "Canceled"),
+                            (5, "Replacement"),
+                            (6, "Duplicated"),
+                            (7, "Deleted"),
+                        ],
+                        default=0,
+                        verbose_name="Schedule relationship",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Trip update",
+                "verbose_name_plural": "Trip updates",
+                "ordering": ("start_date", "trip"),
+                "unique_together": {("trip", "start_date", "start_time")},
+            },
+        ),
+        migrations.CreateModel(
+            name="StopTimeUpdate",
+            fields=[
+                (
+                    "stop_time",
+                    models.OneToOneField(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        primary_key=True,
+                        related_name="update",
+                        serialize=False,
+                        to="sncfgtfs.stoptime",
+                        verbose_name="Stop time",
+                    ),
+                ),
+                ("arrival_delay", models.DurationField(verbose_name="Arrival delay")),
+                ("arrival_time", models.DateTimeField(verbose_name="Arrival time")),
+                (
+                    "departure_delay",
+                    models.DurationField(verbose_name="Departure delay"),
+                ),
+                ("departure_time", models.DateTimeField(verbose_name="Departure time")),
+                (
+                    "schedule_relationship",
+                    models.IntegerField(
+                        choices=[
+                            (0, "Scheduled"),
+                            (1, "Skipped"),
+                            (2, "No data"),
+                            (3, "Unscheduled"),
+                        ],
+                        default=0,
+                        verbose_name="Schedule relationship",
+                    ),
+                ),
+                (
+                    "trip_update",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="stop_time_updates",
+                        to="sncfgtfs.tripupdate",
+                        verbose_name="Trip update",
+                    ),
+                ),
+            ],
+            options={
+                "verbose_name": "Stop time update",
+                "verbose_name_plural": "Stop time updates",
+                "ordering": ("trip_update", "stop_time"),
+                "unique_together": {("trip_update", "stop_time")},
+            },
+        ),
     ]

sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py

@@ -1,108 +0,0 @@
-# Generated by Django 5.0.1 on 2024-02-06 06:59
-
-import django.db.models.deletion
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("sncfgtfs", "0001_initial"),
-    ]
-
-    operations = [
-        migrations.AlterModelOptions(
-            name="trip",
-            options={"verbose_name": "Trip", "verbose_name_plural": "Trips"},
-        ),
-        migrations.CreateModel(
-            name="TripUpdate",
-            fields=[
-                (
-                    "trip",
-                    models.OneToOneField(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        primary_key=True,
-                        related_name="update",
-                        serialize=False,
-                        to="sncfgtfs.trip",
-                        verbose_name="Trip",
-                    ),
-                ),
-                ("start_date", models.DateField(verbose_name="Start date")),
-                ("start_time", models.TimeField(verbose_name="Start time")),
-                (
-                    "schedule_relationship",
-                    models.IntegerField(
-                        choices=[
-                            (0, "Scheduled"),
-                            (1, "Added"),
-                            (2, "Unscheduled"),
-                            (3, "Canceled"),
-                            (5, "Replacement"),
-                            (6, "Duplicated"),
-                            (7, "Deleted"),
-                        ],
-                        default=0,
-                        verbose_name="Schedule relationship",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "Trip update",
-                "verbose_name_plural": "Trip updates",
-                "ordering": ("start_date", "trip"),
-                "unique_together": {("trip", "start_date", "start_time")},
-            },
-        ),
-        migrations.CreateModel(
-            name="StopTimeUpdate",
-            fields=[
-                (
-                    "stop_time",
-                    models.OneToOneField(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        primary_key=True,
-                        related_name="update",
-                        serialize=False,
-                        to="sncfgtfs.stoptime",
-                        verbose_name="Stop time",
-                    ),
-                ),
-                ("arrival_delay", models.DurationField(verbose_name="Arrival delay")),
-                ("arrival_time", models.DateTimeField(verbose_name="Arrival time")),
-                (
-                    "departure_delay",
-                    models.DurationField(verbose_name="Departure delay"),
-                ),
-                ("departure_time", models.DateTimeField(verbose_name="Departure time")),
-                (
-                    "schedule_relationship",
-                    models.IntegerField(
-                        choices=[
-                            (0, "Scheduled"),
-                            (1, "Skipped"),
-                            (2, "No data"),
-                            (3, "Unscheduled"),
-                        ],
-                        default=0,
-                        verbose_name="Schedule relationship",
-                    ),
-                ),
-                (
-                    "trip_update",
-                    models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE,
-                        related_name="stop_time_updates",
-                        to="sncfgtfs.tripupdate",
-                        verbose_name="Trip update",
-                    ),
-                ),
-            ],
-            options={
-                "verbose_name": "Stop time update",
-                "verbose_name_plural": "Stop time updates",
-                "ordering": ("trip_update", "stop_time"),
-                "unique_together": {("trip_update", "stop_time")},
-            },
-        ),
-    ]

sncfgtfs/models.py

@@ -230,7 +230,7 @@ class Route(models.Model):
     agency = models.ForeignKey(
         to="Agency",
         on_delete=models.CASCADE,
-        verbose_name=_("Agency ID"),
+        verbose_name=_("Agency"),
         related_name="routes",
     )
 
@@ -272,6 +272,12 @@ class Route(models.Model):
         blank=True,
     )
 
+    transport_type = models.CharField(
+        max_length=255,
+        verbose_name=_("Transport type"),
+        choices=TransportType,
+    )
+
     def __str__(self):
         return f"{self.long_name}"
 
@@ -346,6 +352,11 @@ class Trip(models.Model):
         null=True,
     )
 
+    last_update = models.DateTimeField(
+        verbose_name=_("Last update"),
+        null=True,
+    )
+
     @property
     def origin(self):
         return self.stop_times.order_by('stop_sequence').first().stop
@@ -354,16 +365,30 @@ class Trip(models.Model):
     def destination(self):
         return self.stop_times.order_by('-stop_sequence').first().stop
 
+    @property
+    def departure_time(self):
+        dep_time = self.stop_times.order_by('stop_sequence').first().departure_time
+        hours = int(dep_time.total_seconds() // 3600)
+        minutes = int((dep_time.total_seconds() % 3600) // 60)
+        return f"{hours:02}:{minutes:02}"
+
+    @property
+    def arrival_time(self):
+        arr_time = self.stop_times.order_by('-stop_sequence').first().arrival_time
+        hours = int(arr_time.total_seconds() // 3600)
+        minutes = int((arr_time.total_seconds() % 3600) // 60)
+        return f"{hours:02}:{minutes:02}"
+
     @property
     def train_type(self):
-        if self.service.transport_type == TransportType.TRANSILIEN:
+        if self.route.transport_type == TransportType.TRANSILIEN:
             return self.route.short_name
         else:
             return self.origin.stop_type
 
     @property
     def train_number(self):
-        if self.service.transport_type == TransportType.TRANSILIEN:
+        if self.route.transport_type == TransportType.TRANSILIEN:
             return self.short_name
         else:
             return self.headsign
@@ -389,7 +414,8 @@ class Trip(models.Model):
         return "000000"
 
     def __str__(self):
-        return f"{self.route.long_name} - {self.id}"
+        return f"{self.origin.name} {self.departure_time} → {self.destination.name} {self.arrival_time}" \
+               f" - {self.service_id}"
 
     class Meta:
         verbose_name = _("Trip")
@@ -470,7 +496,7 @@ class StopTime(models.Model):
         return f"{hours:02}:{minutes:02}"
 
     def __str__(self):
-        return f"{self.trip.route.long_name} - {self.trip_id} - {self.stop.name}"
+        return f"{self.stop.name} - {self.trip_id}"
 
     class Meta:
         verbose_name = _("Stop time")
@@ -558,12 +584,6 @@ class CalendarDate(models.Model):
         choices=ExceptionType,
     )
 
-    transport_type = models.CharField(
-        max_length=255,
-        verbose_name=_("Transport type"),
-        choices=TransportType,
-    )
-
     def __str__(self):
         return f"{self.service.id} - {self.date} - {self.exception_type}"