ynerant
/
trainvel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Truncate trip id

This commit is contained in:
Emmy D'Anello 2024-02-09 23:15:14 +01:00
parent 8d2ffe3014
commit 77c3ef9e74
Signed by: ynerant
GPG Key ID: 3A75C55819C8CF85
5 changed files with 225 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
sncfgtfs/admin.py
View File

@ -4,6 +4,37 @@ from sncfgtfs.models import Agency, Stop, Route, Trip, StopTime, Calendar, Calen
Transfer, FeedInfo, StopTimeUpdate, TripUpdate
class CalendarDateInline(admin.TabularInline):
model = CalendarDate
extra = 0
class TripInline(admin.TabularInline):
model = Trip
extra = 0
autocomplete_fields = ('route', 'service',)
show_change_link = True
ordering = ('service',)
class StopTimeInline(admin.TabularInline):
model = StopTime
extra = 0
autocomplete_fields = ('stop',)
show_change_link = True
ordering = ('stop_sequence',)
class TripUpdateInline(admin.StackedInline):
model = TripUpdate
extra = 0
class StopTimeUpdateInline(admin.StackedInline):
model = StopTimeUpdate
extra = 0
@admin.register(Agency)
class AgencyAdmin(admin.ModelAdmin):
list_display = ('name', 'id', 'url', 'timezone',)
@ -26,24 +57,28 @@ class RouteAdmin(admin.ModelAdmin):
search_fields = ('long_name', 'short_name', 'id',)
ordering = ('long_name',)
autocomplete_fields = ('agency',)
inlines = (TripInline,)
@admin.register(Trip)
class TripAdmin(admin.ModelAdmin):
list_display = ('id', 'route', 'service', 'headsign', 'direction_id',)
list_filter = ('direction_id', 'service__transport_type',)
list_filter = ('direction_id', 'route__transport_type',)
search_fields = ('id', 'route__id', 'route__long_name', 'service__id', 'headsign',)
ordering = ('route', 'service',)
autocomplete_fields = ('route', 'service',)
inlines = (StopTimeInline, TripUpdateInline,)
@admin.register(StopTime)
class StopTimeAdmin(admin.ModelAdmin):
list_display = ('trip', 'stop', 'arrival_time', 'departure_time',
'stop_sequence', 'pickup_type', 'drop_off_type',)
list_filter = ('pickup_type', 'drop_off_type', 'trip__service__transport_type',)
list_filter = ('pickup_type', 'drop_off_type', 'trip__route__transport_type',)
search_fields = ('trip__id', 'stop__name', 'arrival_time', 'departure_time',)
ordering = ('trip', 'stop_sequence',)
autocomplete_fields = ('trip', 'stop',)
inlines = (StopTimeUpdateInline,)
@admin.register(Calendar)
@ -54,6 +89,7 @@ class CalendarAdmin(admin.ModelAdmin):
'start_date', 'end_date',)
search_fields = ('id', 'start_date', 'end_date',)
ordering = ('transport_type', 'id',)
inlines = (CalendarDateInline, TripInline,)
@admin.register(CalendarDate)

68
sncfgtfs/management/commands/update_sncf_gtfs.py
View File

@ -20,10 +20,17 @@ class Command(BaseCommand):
}
def add_arguments(self, parser):
parser.add_argument('--bulk_size', type=int, default=1000, help='Number of objects to create in bulk.')
parser.add_argument('--bulk_size', type=int, default=1000, help="Number of objects to create in bulk.")
parser.add_argument('--dry-run', action='store_true',
help="Do not update the database, only print what would be done.")
parser.add_argument('--force', '-f', action='store_true', help="Force the update of the database.")
def handle(self, *args, **options):
bulk_size = options['bulk_size']
dry_run = options['dry_run']
force = options['force']
if dry_run:
self.stdout.write(self.style.WARNING("Dry run mode activated."))
if not FeedInfo.objects.exists():
last_update_date = "1970-01-01"
@ -36,11 +43,14 @@ class Command(BaseCommand):
if last_modified.date().isoformat() > last_update_date:
break
else:
self.stdout.write(self.style.WARNING("Database already up-to-date."))
return
if not force:
self.stdout.write(self.style.WARNING("Database already up-to-date."))
return
self.stdout.write("Updating database...")
all_trips = []
for transport_type, feed_url in self.GTFS_FEEDS.items():
self.stdout.write(f"Downloading {transport_type} GTFS feed...")
with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile:
@ -57,7 +67,7 @@ class Command(BaseCommand):
email=agency_dict.get('agency_email', ""),
)
agencies.append(agency)
if agencies:
if agencies and not dry_run:
Agency.objects.bulk_create(agencies,
update_conflicts=True,
update_fields=['name', 'url', 'timezone', 'lang', 'phone', 'email'],
@ -85,7 +95,7 @@ class Command(BaseCommand):
)
stops.append(stop)
if len(stops) >= bulk_size:
if len(stops) >= bulk_size and not dry_run:
Stop.objects.bulk_create(stops,
update_conflicts=True,
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
@ -93,7 +103,7 @@ class Command(BaseCommand):
'wheelchair_boarding', 'level_id', 'platform_code'],
unique_fields=['id'])
stops.clear()
if stops:
if stops and not dry_run:
Stop.objects.bulk_create(stops,
update_conflicts=True,
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
@ -115,17 +125,19 @@ class Command(BaseCommand):
url=route_dict['route_url'],
color=route_dict['route_color'],
text_color=route_dict['route_text_color'],
transport_type=transport_type,
)
routes.append(route)
if len(routes) >= bulk_size:
if len(routes) >= bulk_size and not dry_run:
Route.objects.bulk_create(routes,
update_conflicts=True,
update_fields=['agency_id', 'short_name', 'long_name', 'desc',
'type', 'url', 'color', 'text_color'],
'type', 'url', 'color', 'text_color',
'transport_type'],
unique_fields=['id'])
routes.clear()
if routes:
if routes and not dry_run:
Route.objects.bulk_create(routes,
update_conflicts=True,
update_fields=['agency_id', 'short_name', 'long_name', 'desc',
@ -154,7 +166,7 @@ class Command(BaseCommand):
calendars.append(calendar)
calendar_ids.append(calendar.id)
if len(calendars) >= bulk_size:
if len(calendars) >= bulk_size and not dry_run:
Calendar.objects.bulk_create(calendars,
update_conflicts=True,
update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
@ -162,7 +174,7 @@ class Command(BaseCommand):
'end_date', 'transport_type'],
unique_fields=['id'])
calendars.clear()
if calendars:
if calendars and not dry_run:
Calendar.objects.bulk_create(calendars, update_conflicts=True,
update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
'friday', 'saturday', 'sunday', 'start_date',
@ -198,7 +210,7 @@ class Command(BaseCommand):
)
calendars.append(calendar)
if len(calendar_dates) >= bulk_size:
if len(calendar_dates) >= bulk_size and not dry_run:
Calendar.objects.bulk_create(calendars,
update_conflicts=True,
update_fields=['end_date'],
@ -210,7 +222,7 @@ class Command(BaseCommand):
calendars.clear()
calendar_dates.clear()
if calendar_dates:
if calendar_dates and not dry_run:
Calendar.objects.bulk_create(calendars,
update_conflicts=True,
update_fields=['end_date'],
@ -225,8 +237,14 @@ class Command(BaseCommand):
trips = []
for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()):
trip_dict: dict
trip_id = trip_dict['trip_id']
if transport_type != "TN":
trip_id, last_update = trip_id.split(':', 1)
last_update = datetime.fromisoformat(last_update)
else:
last_update = None
trip = Trip(
id=trip_dict['trip_id'],
id=trip_id,
route_id=trip_dict['route_id'],
service_id=f"{transport_type}-{trip_dict['service_id']}",
headsign=trip_dict['trip_headsign'],
@ -236,10 +254,11 @@ class Command(BaseCommand):
shape_id=trip_dict['shape_id'],
wheelchair_accessible=trip_dict.get('wheelchair_accessible', None),
bikes_allowed=trip_dict.get('bikes_allowed', None),
last_update=last_update,
)
trips.append(trip)
if len(trips) >= bulk_size:
if len(trips) >= bulk_size and not dry_run:
Trip.objects.bulk_create(trips,
update_conflicts=True,
update_fields=['route_id', 'service_id', 'headsign', 'short_name',
@ -247,7 +266,7 @@ class Command(BaseCommand):
'wheelchair_accessible', 'bikes_allowed'],
unique_fields=['id'])
trips.clear()
if trips:
if trips and not dry_run:
Trip.objects.bulk_create(trips,
update_conflicts=True,
update_fields=['route_id', 'service_id', 'headsign', 'short_name',
@ -256,17 +275,22 @@ class Command(BaseCommand):
unique_fields=['id'])
trips.clear()
all_trips.extend(trips)
stop_times = []
for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()):
stop_time_dict: dict
trip_id = stop_time_dict['trip_id']
if transport_type != "TN":
trip_id = trip_id.split(':', 1)[0]
arr_time = stop_time_dict['arrival_time']
arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:])
dep_time = stop_time_dict['departure_time']
dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:])
st = StopTime(
id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}",
trip_id=stop_time_dict['trip_id'],
trip_id=trip_id,
arrival_time=timedelta(seconds=arr_time),
departure_time=timedelta(seconds=dep_time),
stop_id=stop_time_dict['stop_id'],
@ -278,7 +302,7 @@ class Command(BaseCommand):
)
stop_times.append(st)
if len(stop_times) >= bulk_size:
if len(stop_times) >= bulk_size and not dry_run:
StopTime.objects.bulk_create(stop_times,
update_conflicts=True,
update_fields=['stop_id', 'arrival_time', 'departure_time',
@ -286,7 +310,7 @@ class Command(BaseCommand):
'drop_off_type', 'timepoint'],
unique_fields=['id'])
stop_times.clear()
if stop_times:
if stop_times and not dry_run:
StopTime.objects.bulk_create(stop_times,
update_conflicts=True,
update_fields=['stop_id', 'arrival_time', 'departure_time',
@ -307,21 +331,21 @@ class Command(BaseCommand):
)
transfers.append(transfer)
if len(transfers) >= bulk_size:
if len(transfers) >= bulk_size and not dry_run:
Transfer.objects.bulk_create(transfers,
update_conflicts=True,
update_fields=['transfer_type', 'min_transfer_time'],
unique_fields=['id'])
transfers.clear()
if transfers:
if transfers and not dry_run:
Transfer.objects.bulk_create(transfers,
update_conflicts=True,
update_fields=['transfer_type', 'min_transfer_time'],
unique_fields=['id'])
transfers.clear()
if "feed_info.txt" in zipfile.namelist():
if "feed_info.txt" in zipfile.namelist() and not dry_run:
for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()):
feed_info_dict: dict
FeedInfo.objects.update_or_create(

126
sncfgtfs/migrations/0001_initial.py
View File

@ -1,4 +1,4 @@
# Generated by Django 5.0.1 on 2024-01-27 14:08
# Generated by Django 5.0.1 on 2024-02-09 21:55
import django.db.models.deletion
from django.db import migrations, models
@ -155,19 +155,6 @@ class Migration(migrations.Migration):
verbose_name="Exception type",
),
),
(
"transport_type",
models.CharField(
choices=[
("TGV", "TGV"),
("TER", "TER"),
("IC", "Intercités"),
("TN", "Transilien"),
],
max_length=255,
verbose_name="Transport type",
),
),
(
"service",
models.ForeignKey(
@ -239,13 +226,26 @@ class Migration(migrations.Migration):
blank=True, max_length=255, verbose_name="Route text color"
),
),
(
"transport_type",
models.CharField(
choices=[
("TGV", "TGV"),
("TER", "TER"),
("IC", "Intercités"),
("TN", "Transilien"),
],
max_length=255,
verbose_name="Transport type",
),
),
(
"agency",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="routes",
to="sncfgtfs.agency",
verbose_name="Agency ID",
verbose_name="Agency",
),
),
],
@ -474,6 +474,10 @@ class Migration(migrations.Migration):
verbose_name="Bikes allowed",
),
),
(
"last_update",
models.DateTimeField(null=True, verbose_name="Last update"),
),
(
"route",
models.ForeignKey(
@ -496,7 +500,6 @@ class Migration(migrations.Migration):
options={
"verbose_name": "Trip",
"verbose_name_plural": "Trips",
"ordering": ("id",),
},
),
migrations.CreateModel(
@ -578,4 +581,95 @@ class Migration(migrations.Migration):
"verbose_name_plural": "Stop times",
},
),
migrations.CreateModel(
name="TripUpdate",
fields=[
(
"trip",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
primary_key=True,
related_name="update",
serialize=False,
to="sncfgtfs.trip",
verbose_name="Trip",
),
),
("start_date", models.DateField(verbose_name="Start date")),
("start_time", models.TimeField(verbose_name="Start time")),
(
"schedule_relationship",
models.IntegerField(
choices=[
(0, "Scheduled"),
(1, "Added"),
(2, "Unscheduled"),
(3, "Canceled"),
(5, "Replacement"),
(6, "Duplicated"),
(7, "Deleted"),
],
default=0,
verbose_name="Schedule relationship",
),
),
],
options={
"verbose_name": "Trip update",
"verbose_name_plural": "Trip updates",
"ordering": ("start_date", "trip"),
"unique_together": {("trip", "start_date", "start_time")},
},
),
migrations.CreateModel(
name="StopTimeUpdate",
fields=[
(
"stop_time",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
primary_key=True,
related_name="update",
serialize=False,
to="sncfgtfs.stoptime",
verbose_name="Stop time",
),
),
("arrival_delay", models.DurationField(verbose_name="Arrival delay")),
("arrival_time", models.DateTimeField(verbose_name="Arrival time")),
(
"departure_delay",
models.DurationField(verbose_name="Departure delay"),
),
("departure_time", models.DateTimeField(verbose_name="Departure time")),
(
"schedule_relationship",
models.IntegerField(
choices=[
(0, "Scheduled"),
(1, "Skipped"),
(2, "No data"),
(3, "Unscheduled"),
],
default=0,
verbose_name="Schedule relationship",
),
),
(
"trip_update",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="stop_time_updates",
to="sncfgtfs.tripupdate",
verbose_name="Trip update",
),
),
],
options={
"verbose_name": "Stop time update",
"verbose_name_plural": "Stop time updates",
"ordering": ("trip_update", "stop_time"),
"unique_together": {("trip_update", "stop_time")},
},
),
]

108
sncfgtfs/migrations/0002_alter_trip_options_tripupdate_stoptimeupdate.py
View File

@ -1,108 +0,0 @@
# Generated by Django 5.0.1 on 2024-02-06 06:59
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("sncfgtfs", "0001_initial"),
]
operations = [
migrations.AlterModelOptions(
name="trip",
options={"verbose_name": "Trip", "verbose_name_plural": "Trips"},
),
migrations.CreateModel(
name="TripUpdate",
fields=[
(
"trip",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
primary_key=True,
related_name="update",
serialize=False,
to="sncfgtfs.trip",
verbose_name="Trip",
),
),
("start_date", models.DateField(verbose_name="Start date")),
("start_time", models.TimeField(verbose_name="Start time")),
(
"schedule_relationship",
models.IntegerField(
choices=[
(0, "Scheduled"),
(1, "Added"),
(2, "Unscheduled"),
(3, "Canceled"),
(5, "Replacement"),
(6, "Duplicated"),
(7, "Deleted"),
],
default=0,
verbose_name="Schedule relationship",
),
),
],
options={
"verbose_name": "Trip update",
"verbose_name_plural": "Trip updates",
"ordering": ("start_date", "trip"),
"unique_together": {("trip", "start_date", "start_time")},
},
),
migrations.CreateModel(
name="StopTimeUpdate",
fields=[
(
"stop_time",
models.OneToOneField(
on_delete=django.db.models.deletion.CASCADE,
primary_key=True,
related_name="update",
serialize=False,
to="sncfgtfs.stoptime",
verbose_name="Stop time",
),
),
("arrival_delay", models.DurationField(verbose_name="Arrival delay")),
("arrival_time", models.DateTimeField(verbose_name="Arrival time")),
(
"departure_delay",
models.DurationField(verbose_name="Departure delay"),
),
("departure_time", models.DateTimeField(verbose_name="Departure time")),
(
"schedule_relationship",
models.IntegerField(
choices=[
(0, "Scheduled"),
(1, "Skipped"),
(2, "No data"),
(3, "Unscheduled"),
],
default=0,
verbose_name="Schedule relationship",
),
),
(
"trip_update",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="stop_time_updates",
to="sncfgtfs.tripupdate",
verbose_name="Trip update",
),
),
],
options={
"verbose_name": "Stop time update",
"verbose_name_plural": "Stop time updates",
"ordering": ("trip_update", "stop_time"),
"unique_together": {("trip_update", "stop_time")},
},
),
]

42
sncfgtfs/models.py
View File

@ -230,7 +230,7 @@ class Route(models.Model):
agency = models.ForeignKey(
to="Agency",
on_delete=models.CASCADE,
verbose_name=_("Agency ID"),
verbose_name=_("Agency"),
related_name="routes",
)
@ -272,6 +272,12 @@ class Route(models.Model):
blank=True,
)
transport_type = models.CharField(
max_length=255,
verbose_name=_("Transport type"),
choices=TransportType,
)
def __str__(self):
return f"{self.long_name}"
@ -346,6 +352,11 @@ class Trip(models.Model):
null=True,
)
last_update = models.DateTimeField(
verbose_name=_("Last update"),
null=True,
)
@property
def origin(self):
return self.stop_times.order_by('stop_sequence').first().stop
@ -354,16 +365,30 @@ class Trip(models.Model):
def destination(self):
return self.stop_times.order_by('-stop_sequence').first().stop
@property
def departure_time(self):
dep_time = self.stop_times.order_by('stop_sequence').first().departure_time
hours = int(dep_time.total_seconds() // 3600)
minutes = int((dep_time.total_seconds() % 3600) // 60)
return f"{hours:02}:{minutes:02}"
@property
def arrival_time(self):
arr_time = self.stop_times.order_by('-stop_sequence').first().arrival_time
hours = int(arr_time.total_seconds() // 3600)
minutes = int((arr_time.total_seconds() % 3600) // 60)
return f"{hours:02}:{minutes:02}"
@property
def train_type(self):
if self.service.transport_type == TransportType.TRANSILIEN:
if self.route.transport_type == TransportType.TRANSILIEN:
return self.route.short_name
else:
return self.origin.stop_type
@property
def train_number(self):
if self.service.transport_type == TransportType.TRANSILIEN:
if self.route.transport_type == TransportType.TRANSILIEN:
return self.short_name
else:
return self.headsign
@ -389,7 +414,8 @@ class Trip(models.Model):
return "000000"
def __str__(self):
return f"{self.route.long_name} - {self.id}"
return f"{self.origin.name} {self.departure_time} → {self.destination.name} {self.arrival_time}" \
f" - {self.service_id}"
class Meta:
verbose_name = _("Trip")
@ -470,7 +496,7 @@ class StopTime(models.Model):
return f"{hours:02}:{minutes:02}"
def __str__(self):
return f"{self.trip.route.long_name} - {self.trip_id} - {self.stop.name}"
return f"{self.stop.name} - {self.trip_id}"
class Meta:
verbose_name = _("Stop time")
@ -558,12 +584,6 @@ class CalendarDate(models.Model):
choices=ExceptionType,
)
transport_type = models.CharField(
max_length=255,
verbose_name=_("Transport type"),
choices=TransportType,
)
def __str__(self):
return f"{self.service.id} - {self.date} - {self.exception_type}"