Truncate trip id
This commit is contained in:
@ -20,10 +20,17 @@ class Command(BaseCommand):
|
||||
}
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument('--bulk_size', type=int, default=1000, help='Number of objects to create in bulk.')
|
||||
parser.add_argument('--bulk_size', type=int, default=1000, help="Number of objects to create in bulk.")
|
||||
parser.add_argument('--dry-run', action='store_true',
|
||||
help="Do not update the database, only print what would be done.")
|
||||
parser.add_argument('--force', '-f', action='store_true', help="Force the update of the database.")
|
||||
|
||||
def handle(self, *args, **options):
|
||||
bulk_size = options['bulk_size']
|
||||
dry_run = options['dry_run']
|
||||
force = options['force']
|
||||
if dry_run:
|
||||
self.stdout.write(self.style.WARNING("Dry run mode activated."))
|
||||
|
||||
if not FeedInfo.objects.exists():
|
||||
last_update_date = "1970-01-01"
|
||||
@ -36,11 +43,14 @@ class Command(BaseCommand):
|
||||
if last_modified.date().isoformat() > last_update_date:
|
||||
break
|
||||
else:
|
||||
self.stdout.write(self.style.WARNING("Database already up-to-date."))
|
||||
return
|
||||
if not force:
|
||||
self.stdout.write(self.style.WARNING("Database already up-to-date."))
|
||||
return
|
||||
|
||||
self.stdout.write("Updating database...")
|
||||
|
||||
all_trips = []
|
||||
|
||||
for transport_type, feed_url in self.GTFS_FEEDS.items():
|
||||
self.stdout.write(f"Downloading {transport_type} GTFS feed...")
|
||||
with ZipFile(BytesIO(requests.get(feed_url).content)) as zipfile:
|
||||
@ -57,7 +67,7 @@ class Command(BaseCommand):
|
||||
email=agency_dict.get('agency_email', ""),
|
||||
)
|
||||
agencies.append(agency)
|
||||
if agencies:
|
||||
if agencies and not dry_run:
|
||||
Agency.objects.bulk_create(agencies,
|
||||
update_conflicts=True,
|
||||
update_fields=['name', 'url', 'timezone', 'lang', 'phone', 'email'],
|
||||
@ -85,7 +95,7 @@ class Command(BaseCommand):
|
||||
)
|
||||
stops.append(stop)
|
||||
|
||||
if len(stops) >= bulk_size:
|
||||
if len(stops) >= bulk_size and not dry_run:
|
||||
Stop.objects.bulk_create(stops,
|
||||
update_conflicts=True,
|
||||
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
|
||||
@ -93,7 +103,7 @@ class Command(BaseCommand):
|
||||
'wheelchair_boarding', 'level_id', 'platform_code'],
|
||||
unique_fields=['id'])
|
||||
stops.clear()
|
||||
if stops:
|
||||
if stops and not dry_run:
|
||||
Stop.objects.bulk_create(stops,
|
||||
update_conflicts=True,
|
||||
update_fields=['name', 'desc', 'lat', 'lon', 'zone_id', 'url',
|
||||
@ -115,17 +125,19 @@ class Command(BaseCommand):
|
||||
url=route_dict['route_url'],
|
||||
color=route_dict['route_color'],
|
||||
text_color=route_dict['route_text_color'],
|
||||
transport_type=transport_type,
|
||||
)
|
||||
routes.append(route)
|
||||
|
||||
if len(routes) >= bulk_size:
|
||||
if len(routes) >= bulk_size and not dry_run:
|
||||
Route.objects.bulk_create(routes,
|
||||
update_conflicts=True,
|
||||
update_fields=['agency_id', 'short_name', 'long_name', 'desc',
|
||||
'type', 'url', 'color', 'text_color'],
|
||||
'type', 'url', 'color', 'text_color',
|
||||
'transport_type'],
|
||||
unique_fields=['id'])
|
||||
routes.clear()
|
||||
if routes:
|
||||
if routes and not dry_run:
|
||||
Route.objects.bulk_create(routes,
|
||||
update_conflicts=True,
|
||||
update_fields=['agency_id', 'short_name', 'long_name', 'desc',
|
||||
@ -154,7 +166,7 @@ class Command(BaseCommand):
|
||||
calendars.append(calendar)
|
||||
calendar_ids.append(calendar.id)
|
||||
|
||||
if len(calendars) >= bulk_size:
|
||||
if len(calendars) >= bulk_size and not dry_run:
|
||||
Calendar.objects.bulk_create(calendars,
|
||||
update_conflicts=True,
|
||||
update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
|
||||
@ -162,7 +174,7 @@ class Command(BaseCommand):
|
||||
'end_date', 'transport_type'],
|
||||
unique_fields=['id'])
|
||||
calendars.clear()
|
||||
if calendars:
|
||||
if calendars and not dry_run:
|
||||
Calendar.objects.bulk_create(calendars, update_conflicts=True,
|
||||
update_fields=['monday', 'tuesday', 'wednesday', 'thursday',
|
||||
'friday', 'saturday', 'sunday', 'start_date',
|
||||
@ -198,7 +210,7 @@ class Command(BaseCommand):
|
||||
)
|
||||
calendars.append(calendar)
|
||||
|
||||
if len(calendar_dates) >= bulk_size:
|
||||
if len(calendar_dates) >= bulk_size and not dry_run:
|
||||
Calendar.objects.bulk_create(calendars,
|
||||
update_conflicts=True,
|
||||
update_fields=['end_date'],
|
||||
@ -210,7 +222,7 @@ class Command(BaseCommand):
|
||||
calendars.clear()
|
||||
calendar_dates.clear()
|
||||
|
||||
if calendar_dates:
|
||||
if calendar_dates and not dry_run:
|
||||
Calendar.objects.bulk_create(calendars,
|
||||
update_conflicts=True,
|
||||
update_fields=['end_date'],
|
||||
@ -225,8 +237,14 @@ class Command(BaseCommand):
|
||||
trips = []
|
||||
for trip_dict in csv.DictReader(zipfile.read("trips.txt").decode().splitlines()):
|
||||
trip_dict: dict
|
||||
trip_id = trip_dict['trip_id']
|
||||
if transport_type != "TN":
|
||||
trip_id, last_update = trip_id.split(':', 1)
|
||||
last_update = datetime.fromisoformat(last_update)
|
||||
else:
|
||||
last_update = None
|
||||
trip = Trip(
|
||||
id=trip_dict['trip_id'],
|
||||
id=trip_id,
|
||||
route_id=trip_dict['route_id'],
|
||||
service_id=f"{transport_type}-{trip_dict['service_id']}",
|
||||
headsign=trip_dict['trip_headsign'],
|
||||
@ -236,10 +254,11 @@ class Command(BaseCommand):
|
||||
shape_id=trip_dict['shape_id'],
|
||||
wheelchair_accessible=trip_dict.get('wheelchair_accessible', None),
|
||||
bikes_allowed=trip_dict.get('bikes_allowed', None),
|
||||
last_update=last_update,
|
||||
)
|
||||
trips.append(trip)
|
||||
|
||||
if len(trips) >= bulk_size:
|
||||
if len(trips) >= bulk_size and not dry_run:
|
||||
Trip.objects.bulk_create(trips,
|
||||
update_conflicts=True,
|
||||
update_fields=['route_id', 'service_id', 'headsign', 'short_name',
|
||||
@ -247,7 +266,7 @@ class Command(BaseCommand):
|
||||
'wheelchair_accessible', 'bikes_allowed'],
|
||||
unique_fields=['id'])
|
||||
trips.clear()
|
||||
if trips:
|
||||
if trips and not dry_run:
|
||||
Trip.objects.bulk_create(trips,
|
||||
update_conflicts=True,
|
||||
update_fields=['route_id', 'service_id', 'headsign', 'short_name',
|
||||
@ -256,17 +275,22 @@ class Command(BaseCommand):
|
||||
unique_fields=['id'])
|
||||
trips.clear()
|
||||
|
||||
all_trips.extend(trips)
|
||||
|
||||
stop_times = []
|
||||
for stop_time_dict in csv.DictReader(zipfile.read("stop_times.txt").decode().splitlines()):
|
||||
stop_time_dict: dict
|
||||
|
||||
trip_id = stop_time_dict['trip_id']
|
||||
if transport_type != "TN":
|
||||
trip_id = trip_id.split(':', 1)[0]
|
||||
arr_time = stop_time_dict['arrival_time']
|
||||
arr_time = int(arr_time[:2]) * 3600 + int(arr_time[3:5]) * 60 + int(arr_time[6:])
|
||||
dep_time = stop_time_dict['departure_time']
|
||||
dep_time = int(dep_time[:2]) * 3600 + int(dep_time[3:5]) * 60 + int(dep_time[6:])
|
||||
st = StopTime(
|
||||
id=f"{stop_time_dict['trip_id']}-{stop_time_dict['stop_id']}",
|
||||
trip_id=stop_time_dict['trip_id'],
|
||||
trip_id=trip_id,
|
||||
arrival_time=timedelta(seconds=arr_time),
|
||||
departure_time=timedelta(seconds=dep_time),
|
||||
stop_id=stop_time_dict['stop_id'],
|
||||
@ -278,7 +302,7 @@ class Command(BaseCommand):
|
||||
)
|
||||
stop_times.append(st)
|
||||
|
||||
if len(stop_times) >= bulk_size:
|
||||
if len(stop_times) >= bulk_size and not dry_run:
|
||||
StopTime.objects.bulk_create(stop_times,
|
||||
update_conflicts=True,
|
||||
update_fields=['stop_id', 'arrival_time', 'departure_time',
|
||||
@ -286,7 +310,7 @@ class Command(BaseCommand):
|
||||
'drop_off_type', 'timepoint'],
|
||||
unique_fields=['id'])
|
||||
stop_times.clear()
|
||||
if stop_times:
|
||||
if stop_times and not dry_run:
|
||||
StopTime.objects.bulk_create(stop_times,
|
||||
update_conflicts=True,
|
||||
update_fields=['stop_id', 'arrival_time', 'departure_time',
|
||||
@ -307,21 +331,21 @@ class Command(BaseCommand):
|
||||
)
|
||||
transfers.append(transfer)
|
||||
|
||||
if len(transfers) >= bulk_size:
|
||||
if len(transfers) >= bulk_size and not dry_run:
|
||||
Transfer.objects.bulk_create(transfers,
|
||||
update_conflicts=True,
|
||||
update_fields=['transfer_type', 'min_transfer_time'],
|
||||
unique_fields=['id'])
|
||||
transfers.clear()
|
||||
|
||||
if transfers:
|
||||
if transfers and not dry_run:
|
||||
Transfer.objects.bulk_create(transfers,
|
||||
update_conflicts=True,
|
||||
update_fields=['transfer_type', 'min_transfer_time'],
|
||||
unique_fields=['id'])
|
||||
transfers.clear()
|
||||
|
||||
if "feed_info.txt" in zipfile.namelist():
|
||||
if "feed_info.txt" in zipfile.namelist() and not dry_run:
|
||||
for feed_info_dict in csv.DictReader(zipfile.read("feed_info.txt").decode().splitlines()):
|
||||
feed_info_dict: dict
|
||||
FeedInfo.objects.update_or_create(
|
||||
|
||||
Reference in New Issue
Block a user