ynerant/trainvel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Prepare code for Eurostar/Trenitalia France/RENFE/ÖBB data input

Browse Source
This commit is contained in:
Emmy D'Anello
2024-02-10 17:33:36 +01:00
parent 77c3ef9e74
commit 16520c3664
19 changed files with 463 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
sncfgtfs/admin.py
View File

@ -44,7 +44,7 @@ class AgencyAdmin(admin.ModelAdmin):
@admin.register(Stop)
class StopAdmin(admin.ModelAdmin):
list_display = ('name', 'id', 'lat', 'lon', 'location_type',)
list_filter = ('location_type',)
list_filter = ('location_type', 'transport_type',)
search_fields = ('name', 'id',)
ordering = ('name',)
autocomplete_fields = ('parent_station',)
@ -52,8 +52,8 @@ class StopAdmin(admin.ModelAdmin):
@admin.register(Route)
class RouteAdmin(admin.ModelAdmin):
list_display = ('long_name', 'short_name', 'id', 'type',)
list_filter = ('type',)
list_display = ('short_name', 'long_name', 'id', 'type',)
list_filter = ('transport_type', 'type', 'agency',)
search_fields = ('long_name', 'short_name', 'id',)
ordering = ('long_name',)
autocomplete_fields = ('agency',)

185
sncfgtfs/management/commands/update_sncf_gtfs.py
View File

@ -17,6 +17,10 @@ class Command(BaseCommand):
"IC": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-intercites-gtfs-last.zip",
"TER": "https://eu.ftp.opendatasoft.com/sncf/gtfs/export-ter-gtfs-last.zip",
"TN": "https://eu.ftp.opendatasoft.com/sncf/gtfs/transilien-gtfs.zip",
# "ES": "https://www.data.gouv.fr/fr/datasets/r/9089b550-696e-4ae0-87b5-40ea55a14292",
# "TI": "https://www.data.gouv.fr/fr/datasets/r/4d1dd21a-b061-47ac-9514-57ffcc09b4a5",
# "RENFE": "https://ssl.renfe.com/gtransit/Fichero_AV_LD/google_transit.zip",
# "OBB": "https://static.oebb.at/open-data/soll-fahrplan-gtfs/GTFS_OP_2024_obb.zip",
}
def add_arguments(self, parser):
@ -35,10 +39,13 @@ class Command(BaseCommand):
if not FeedInfo.objects.exists():
last_update_date = "1970-01-01"
else:
last_update_date = FeedInfo.objects.get().version
last_update_date = FeedInfo.objects.get(publisher_name='SNCF_default').version
for url in self.GTFS_FEEDS.values():
last_modified = requests.head(url).headers["Last-Modified"]
resp = requests.head(url)
if "Last-Modified" not in resp.headers:
continue
last_modified = resp.headers["Last-Modified"]
last_modified = datetime.strptime(last_modified, "%a, %d %b %Y %H:%M:%S %Z")
if last_modified.date().isoformat() > last_update_date:
break
@ -54,15 +61,22 @@ class Command(BaseCommand):
for transport_type, feed_url in self.GTFS_FEEDS.items():
self.stdout.write(f"Downloading {transport_type} GTFS feed...")
with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile:
def read_file(filename):
lines = zipfile.read(filename).decode().replace('\ufeff', '').splitlines()
return [line.strip() for line in lines]
agencies = []
for agency_dict in csv.DictReader(zipfile.read("agency.txt").decode().splitlines()):
for agency_dict in csv.DictReader(read_file("agency.txt")):
agency_dict: dict
if transport_type == "ES" \
and agency_dict['agency_id'] != 'ES' and agency_dict['agency_id'] != 'ER':
continue
agency = Agency(
id=agency_dict['agency_id'],
name=agency_dict['agency_name'],
url=agency_dict['agency_url'],
timezone=agency_dict['agency_timezone'],
lang=agency_dict['agency_lang'],
lang=agency_dict.get('agency_lang', "fr"),
phone=agency_dict.get('agency_phone', ""),
email=agency_dict.get('agency_email', ""),
)
@ -75,23 +89,28 @@ class Command(BaseCommand):
agencies.clear()
stops = []
for stop_dict in csv.DictReader(zipfile.read("stops.txt").decode().splitlines()):
for stop_dict in csv.DictReader(read_file("stops.txt")):
stop_dict: dict
stop_id = stop_dict['stop_id']
if transport_type in ["ES", "TI", "RENFE"]:
stop_id = f"{transport_type}-{stop_id}"
stop = Stop(
id=stop_dict["stop_id"],
id=stop_id,
name=stop_dict['stop_name'],
desc=stop_dict['stop_desc'],
desc=stop_dict.get('stop_desc', ""),
lat=stop_dict['stop_lat'],
lon=stop_dict['stop_lon'],
zone_id=stop_dict['zone_id'],
url=stop_dict['stop_url'],
location_type=stop_dict['location_type'],
parent_station_id=stop_dict['parent_station'] or None
zone_id=stop_dict.get('zone_id', ""),
url=stop_dict.get('stop_url', ""),
location_type=stop_dict.get('location_type', 1) or 1,
parent_station_id=stop_dict.get('parent_station', None) or None
if last_update_date != "1970-01-01" or transport_type != "TN" else None,
timezone=stop_dict.get('stop_timezone', ""),
wheelchair_boarding=stop_dict.get('wheelchair_boarding', 0),
level_id=stop_dict.get('level_id', ""),
platform_code=stop_dict.get('platform_code', ""),
transport_type=transport_type,
)
stops.append(stop)
@ -100,7 +119,8 @@ class Command(BaseCommand):
update_conflicts=True,
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
'location_type', 'parent_station_id', 'timezone',
'wheelchair_boarding', 'level_id', 'platform_code'],
'wheelchair_boarding', 'level_id', 'platform_code',
'transport_type'],
unique_fields=['id'])
stops.clear()
if stops and not dry_run:
@ -108,23 +128,27 @@ class Command(BaseCommand):
update_conflicts=True,
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
'location_type', 'parent_station_id', 'timezone',
'wheelchair_boarding', 'level_id', 'platform_code'],
'wheelchair_boarding', 'level_id', 'platform_code',
'transport_type'],
unique_fields=['id'])
stops.clear()
routes = []
for route_dict in csv.DictReader(zipfile.read("routes.txt").decode().splitlines()):
for route_dict in csv.DictReader(read_file("routes.txt")):
route_dict: dict
route_id = route_dict['route_id']
if transport_type == "TI":
route_id = f"{transport_type}-{route_id}"
route = Route(
id=route_dict['route_id'],
id=route_id,
agency_id=route_dict['agency_id'],
short_name=route_dict['route_short_name'],
long_name=route_dict['route_long_name'],
desc=route_dict['route_desc'],
desc=route_dict.get('route_desc', ""),
type=route_dict['route_type'],
url=route_dict['route_url'],
color=route_dict['route_color'],
text_color=route_dict['route_text_color'],
url=route_dict.get('route_url', ""),
color=route_dict.get('route_color', ""),
text_color=route_dict.get('route_text_color', ""),
transport_type=transport_type,
)
routes.append(route)
@ -141,14 +165,15 @@ class Command(BaseCommand):
Route.objects.bulk_create(routes,
update_conflicts=True,
update_fields=['agency_id', 'short_name', 'long_name', 'desc',
'type', 'url', 'color', 'text_color'],
'type', 'url', 'color', 'text_color',
'transport_type'],
unique_fields=['id'])
routes.clear()
calendar_ids = []
if "calendar.txt" in zipfile.namelist():
calendars = []
for calendar_dict in csv.DictReader(zipfile.read("calendar.txt").decode().splitlines()):
for calendar_dict in csv.DictReader(read_file("calendar.txt")):
calendar_dict: dict
calendar = Calendar(
id=f"{transport_type}-{calendar_dict['service_id']}",
@ -184,7 +209,7 @@ class Command(BaseCommand):
calendars = []
calendar_dates = []
for calendar_date_dict in csv.DictReader(zipfile.read("calendar_dates.txt").decode().splitlines()):
for calendar_date_dict in csv.DictReader(read_file("calendar_dates.txt")):
calendar_date_dict: dict
calendar_date = CalendarDate(
id=f"{transport_type}-{calendar_date_dict['service_id']}-{calendar_date_dict['date']}",
@ -235,23 +260,31 @@ class Command(BaseCommand):
calendar_dates.clear()
trips = []
for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()):
for trip_dict in csv.DictReader(read_file("trips.txt")):
trip_dict: dict
trip_id = trip_dict['trip_id']
if transport_type != "TN":
route_id = trip_dict['route_id']
if transport_type in ["TGV", "IC", "TER"]:
trip_id, last_update = trip_id.split(':', 1)
last_update = datetime.fromisoformat(last_update)
elif transport_type in ["ES", "RENFE"]:
trip_id = f"{transport_type}-{trip_id}"
last_update = None
elif transport_type == "TI":
trip_id = f"{transport_type}-{trip_id}"
route_id = f"{transport_type}-{route_id}"
last_update = None
else:
last_update = None
trip = Trip(
id=trip_id,
route_id=trip_dict['route_id'],
route_id=route_id,
service_id=f"{transport_type}-{trip_dict['service_id']}",
headsign=trip_dict['trip_headsign'],
headsign=trip_dict.get('trip_headsign', ""),
short_name=trip_dict.get('trip_short_name', ""),
direction_id=trip_dict['direction_id'] or None,
block_id=trip_dict['block_id'],
shape_id=trip_dict['shape_id'],
direction_id=trip_dict.get('direction_id', None) or None,
block_id=trip_dict.get('block_id', ""),
shape_id=trip_dict.get('shape_id', ""),
wheelchair_accessible=trip_dict.get('wheelchair_accessible', None),
bikes_allowed=trip_dict.get('bikes_allowed', None),
last_update=last_update,
@ -278,26 +311,49 @@ class Command(BaseCommand):
all_trips.extend(trips)
stop_times = []
for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()):
for stop_time_dict in csv.DictReader(read_file("stop_times.txt")):
stop_time_dict: dict
stop_id = stop_time_dict['stop_id']
if transport_type in ["ES", "TI", "RENFE"]:
stop_id = f"{transport_type}-{stop_id}"
trip_id = stop_time_dict['trip_id']
if transport_type != "TN":
if transport_type in ["TGV", "IC", "TER"]:
trip_id = trip_id.split(':', 1)[0]
elif transport_type in ["ES", "TI", "RENFE"]:
trip_id = f"{transport_type}-{trip_id}"
arr_time = stop_time_dict['arrival_time']
arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:])
arr_h, arr_m, arr_s = map(int, arr_time.split(':'))
arr_time = arr_h * 3600 + arr_m * 60 + arr_s
dep_time = stop_time_dict['departure_time']
dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:])
dep_h, dep_m, dep_s = map(int, dep_time.split(':'))
dep_time = dep_h * 3600 + dep_m * 60 + dep_s
pickup_type = stop_time_dict.get('pickup_type', 0)
drop_off_type = stop_time_dict.get('drop_off_type', 0)
if transport_type in ["ES", "RENFE", "OBB"]:
if stop_time_dict['stop_sequence'] == "1":
drop_off_type = 1
elif arr_time == dep_time:
pickup_type = 1
elif transport_type == "TI":
if stop_time_dict['stop_sequence'] == "0":
drop_off_type = 1
elif arr_time == dep_time:
pickup_type = 1
st = StopTime(
id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}",
id=f"{trip_id}-{stop_id}",
trip_id=trip_id,
arrival_time=timedelta(seconds=arr_time),
departure_time=timedelta(seconds=dep_time),
stop_id=stop_time_dict['stop_id'],
stop_id=stop_id,
stop_sequence=stop_time_dict['stop_sequence'],
stop_headsign=stop_time_dict['stop_headsign'],
pickup_type=stop_time_dict['pickup_type'],
drop_off_type=stop_time_dict['drop_off_type'],
stop_headsign=stop_time_dict.get('stop_headsign', ""),
pickup_type=pickup_type,
drop_off_type=drop_off_type,
timepoint=stop_time_dict.get('timepoint', None),
)
stop_times.append(st)
@ -319,42 +375,49 @@ class Command(BaseCommand):
unique_fields=['id'])
stop_times.clear()
transfers = []
for transfer_dict in csv.DictReader(zipfile.read("transfers.txt").decode().splitlines()):
transfer_dict: dict
transfer = Transfer(
id=f"{transfer_dict['from_stop_id']}-{transfer_dict['to_stop_id']}",
from_stop_id=transfer_dict['from_stop_id'],
to_stop_id=transfer_dict['to_stop_id'],
transfer_type=transfer_dict['transfer_type'],
min_transfer_time=transfer_dict['min_transfer_time'],
)
transfers.append(transfer)
if "transfers.txt" in zipfile.namelist():
transfers = []
for transfer_dict in csv.DictReader(read_file("transfers.txt")):
transfer_dict: dict
from_stop_id = transfer_dict['from_stop_id']
to_stop_id = transfer_dict['to_stop_id']
if transport_type in ["ES", "RENFE", "OBB"]:
from_stop_id = f"{transport_type}-{from_stop_id}"
to_stop_id = f"{transport_type}-{to_stop_id}"
if len(transfers) >= bulk_size and not dry_run:
transfer = Transfer(
id=f"{from_stop_id}-{to_stop_id}",
from_stop_id=transfer_dict['from_stop_id'],
to_stop_id=transfer_dict['to_stop_id'],
transfer_type=transfer_dict['transfer_type'],
min_transfer_time=transfer_dict['min_transfer_time'],
)
transfers.append(transfer)
if len(transfers) >= bulk_size and not dry_run:
Transfer.objects.bulk_create(transfers,
update_conflicts=True,
update_fields=['transfer_type', 'min_transfer_time'],
unique_fields=['id'])
transfers.clear()
if transfers and not dry_run:
Transfer.objects.bulk_create(transfers,
update_conflicts=True,
update_fields=['transfer_type', 'min_transfer_time'],
unique_fields=['id'])
transfers.clear()
if transfers and not dry_run:
Transfer.objects.bulk_create(transfers,
update_conflicts=True,
update_fields=['transfer_type', 'min_transfer_time'],
unique_fields=['id'])
transfers.clear()
if "feed_info.txt" in zipfile.namelist() and not dry_run:
for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()):
for feed_info_dict in csv.DictReader(read_file("feed_info.txt")):
feed_info_dict: dict
FeedInfo.objects.update_or_create(
publisher_name=feed_info_dict['feed_publisher_name'],
defaults=dict(
publisher_url=feed_info_dict['feed_publisher_url'],
lang=feed_info_dict['feed_lang'],
start_date=feed_info_dict['feed_start_date'],
end_date=feed_info_dict['feed_end_date'],
version=feed_info_dict['feed_version'],
start_date=feed_info_dict.get('feed_start_date', datetime.now().date()),
end_date=feed_info_dict.get('feed_end_date', datetime.now().date()),
version=feed_info_dict.get('feed_version', 1),
)
)

34
sncfgtfs/migrations/0001_initial.py
View File

@ -1,4 +1,4 @@
# Generated by Django 5.0.1 on 2024-02-09 21:55
# Generated by Django 5.0.1 on 2024-02-10 16:30
import django.db.models.deletion
from django.db import migrations, models
@ -22,12 +22,7 @@ class Migration(migrations.Migration):
verbose_name="Agency ID",
),
),
(
"name",
models.CharField(
max_length=255, unique=True, verbose_name="Agency name"
),
),
("name", models.CharField(max_length=255, verbose_name="Agency name")),
("url", models.URLField(verbose_name="Agency URL")),
(
"timezone",
@ -87,6 +82,10 @@ class Migration(migrations.Migration):
("TER", "TER"),
("IC", "Intercités"),
("TN", "Transilien"),
("ES", "Eurostar"),
("TI", "Trenitalia"),
("RENFE", "Renfe"),
("OBB", "ÖBB"),
],
max_length=255,
verbose_name="Transport type",
@ -234,6 +233,10 @@ class Migration(migrations.Migration):
("TER", "TER"),
("IC", "Intercités"),
("TN", "Transilien"),
("ES", "Eurostar"),
("TI", "Trenitalia"),
("RENFE", "Renfe"),
("OBB", "ÖBB"),
],
max_length=255,
verbose_name="Transport type",
@ -330,6 +333,23 @@ class Migration(migrations.Migration):
blank=True, max_length=255, verbose_name="Platform code"
),
),
(
"transport_type",
models.CharField(
choices=[
("TGV", "TGV"),
("TER", "TER"),
("IC", "Intercités"),
("TN", "Transilien"),
("ES", "Eurostar"),
("TI", "Trenitalia"),
("RENFE", "Renfe"),
("OBB", "ÖBB"),
],
max_length=255,
verbose_name="Transport type",
),
),
(
"parent_station",
models.ForeignKey(

11
sncfgtfs/models.py
View File

@ -7,6 +7,10 @@ class TransportType(models.TextChoices):
TER = "TER", _("TER")
INTERCITES = "IC", _("Intercités")
TRANSILIEN = "TN", _("Transilien")
EUROSTAR = "ES", _("Eurostar")
TRENITALIA = "TI", _("Trenitalia")
RENFE = "RENFE", _("Renfe")
OBB = "OBB", _("ÖBB")
class LocationType(models.IntegerChoices):
@ -84,7 +88,6 @@ class Agency(models.Model):
name = models.CharField(
max_length=255,
unique=True,
verbose_name=_("Agency name"),
)
@ -206,6 +209,12 @@ class Stop(models.Model):
blank=True,
)
transport_type = models.CharField(
max_length=255,
verbose_name=_("Transport type"),
choices=TransportType,
)
@property
def stop_type(self):
train_type = self.id.split('StopPoint:OCE')[1].split('-')[0]