run jan speeds, runtimes make sense, switch dates for feb

cal-itp · Feb 18, 2025 · 63881e3 · 63881e3
1 parent bc3f8a7
commit 63881e3
Show file tree

Hide file tree

Showing 14 changed files with 88 additions and 6 deletions.
diff --git a/_shared_utils/shared_utils/rt_dates.py b/_shared_utils/shared_utils/rt_dates.py
@@ -74,7 +74,8 @@
     "oct2024g": "2024-10-21",  # additional one-off to capture Amtrak in HQTA
     "nov2024": "2024-11-13",
     "dec2024": "2024-12-11",
-    "jan2025": "2025-01-15"
+    "jan2025": "2025-01-15",
+    "feb2025": "2025-02-12"
 }
 
 y2023_dates = [

diff --git a/gtfs_funnel/Makefile b/gtfs_funnel/Makefile
@@ -48,7 +48,7 @@ funnel_gtfs_single_day:
 all:
 	make funnel_gtfs_single_day
     # update open_data/update_vars.py        
-#	cd ../open_data/ && make create_gtfs_schedule_geospatial_open_data -f Makefile
+	cd ../open_data/ && make create_gtfs_schedule_geospatial_open_data -f Makefile
     # update high_quality_transit_areas/update_vars.py    
 	cd ../high_quality_transit_areas/ && make hqta_data -f Makefile    
     # update rt_segment_speeds/segment_speed_utils/project_vars.py

diff --git a/gtfs_funnel/logs/download_data.log b/gtfs_funnel/logs/download_data.log
@@ -726,3 +726,35 @@
 2025-02-13 09:14:08.345 | INFO     | __main__:download_one_day:29 - # operators to run: 187
 2025-02-13 09:14:08.347 | INFO     | __main__:download_one_day:33 - *********** Download st data ***********
 2025-02-13 09:16:20.955 | INFO     | __main__:download_one_day:56 - execution time: 0:02:14.344677
+2025-02-13 09:31:11.072 | INFO     | __main__:download_one_day:45 - Analysis date: 2025-01-15
+2025-02-13 09:31:12.103 | INFO     | __main__:download_one_day:52 - # operators to run: 227
+2025-02-13 09:31:12.104 | INFO     | __main__:download_one_day:56 - *********** Download trips data ***********
+2025-02-13 09:31:47.391 | INFO     | __main__:download_one_day:86 - execution time: 0:00:36.318819
+2025-02-13 09:32:07.382 | INFO     | __main__:download_one_day:22 - Analysis date: 2025-01-15
+2025-02-13 09:32:08.214 | INFO     | __main__:download_one_day:29 - # operators to run: 227
+2025-02-13 09:32:08.214 | INFO     | __main__:download_one_day:33 - *********** Download stops data ***********
+2025-02-13 09:32:18.045 | INFO     | __main__:download_one_day:64 - execution time: 0:00:10.662186
+2025-02-13 09:32:37.209 | INFO     | __main__:download_one_day:22 - Analysis date: 2025-01-15
+2025-02-13 09:32:38.004 | INFO     | __main__:download_one_day:29 - # operators to run: 227
+2025-02-13 09:32:38.005 | INFO     | __main__:download_one_day:33 - *********** Download routelines data ***********
+2025-02-13 09:34:46.144 | INFO     | __main__:download_one_day:63 - execution time: 0:02:08.935049
+2025-02-13 09:35:06.008 | INFO     | __main__:download_one_day:21 - Analysis date: 2025-01-15
+2025-02-13 09:35:07.724 | INFO     | __main__:download_one_day:29 - # operators to run: 187
+2025-02-13 09:35:07.725 | INFO     | __main__:download_one_day:33 - *********** Download st data ***********
+2025-02-13 09:37:14.671 | INFO     | __main__:download_one_day:56 - execution time: 0:02:08.662570
+2025-02-17 13:17:04.939 | INFO     | __main__:download_one_day:45 - Analysis date: 2025-01-15
+2025-02-17 13:17:13.630 | INFO     | __main__:download_one_day:52 - # operators to run: 227
+2025-02-17 13:17:13.631 | INFO     | __main__:download_one_day:56 - *********** Download trips data ***********
+2025-02-17 13:17:48.976 | INFO     | __main__:download_one_day:86 - execution time: 0:00:44.008843
+2025-02-17 13:18:07.694 | INFO     | __main__:download_one_day:22 - Analysis date: 2025-01-15
+2025-02-17 13:18:09.979 | INFO     | __main__:download_one_day:29 - # operators to run: 227
+2025-02-17 13:18:09.980 | INFO     | __main__:download_one_day:33 - *********** Download stops data ***********
+2025-02-17 13:18:20.104 | INFO     | __main__:download_one_day:64 - execution time: 0:00:12.409400
+2025-02-17 13:18:37.911 | INFO     | __main__:download_one_day:22 - Analysis date: 2025-01-15
+2025-02-17 13:18:40.187 | INFO     | __main__:download_one_day:29 - # operators to run: 227
+2025-02-17 13:18:40.187 | INFO     | __main__:download_one_day:33 - *********** Download routelines data ***********
+2025-02-17 13:20:40.224 | INFO     | __main__:download_one_day:63 - execution time: 0:02:02.311628
+2025-02-17 13:20:58.081 | INFO     | __main__:download_one_day:21 - Analysis date: 2025-01-15
+2025-02-17 13:20:59.764 | INFO     | __main__:download_one_day:29 - # operators to run: 187
+2025-02-17 13:20:59.765 | INFO     | __main__:download_one_day:33 - *********** Download st data ***********
+2025-02-17 13:23:03.938 | INFO     | __main__:download_one_day:56 - execution time: 0:02:05.856553
diff --git a/gtfs_funnel/logs/download_vp_v2.log b/gtfs_funnel/logs/download_vp_v2.log
@@ -462,3 +462,25 @@
 2025-01-16 13:56:18.987 | INFO     | __main__:<module>:137 - execution time: 0:08:02.947545
 2025-02-13 09:16:42.270 | INFO     | __main__:<module>:148 - Analysis date: 2025-01-15
 2025-02-13 09:19:25.285 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 0 to GCS: 0:02:43.014412
+2025-02-13 09:37:36.354 | INFO     | __main__:<module>:148 - Analysis date: 2025-01-15
+2025-02-13 09:40:24.568 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 0 to GCS: 0:02:48.212532
+2025-02-13 09:41:36.607 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 1 to GCS: 0:01:12.038637
+2025-02-13 09:45:59.940 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 2 to GCS: 0:04:23.331885
+2025-02-13 09:48:03.850 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 3 to GCS: 0:02:03.908665
+2025-02-13 09:48:03.850 | INFO     | __main__:<module>:155 - execution time: 0:10:27.495360
+2025-02-13 09:48:44.549 | INFO     | __main__:<module>:97 - Analysis date: 2025-01-15
+2025-02-13 09:48:56.532 | INFO     | __main__:<module>:105 - concat and filter batched data: 0:00:11.982209
+2025-02-13 09:53:01.735 | INFO     | __main__:<module>:112 - export concatenated vp: 0:04:05.202995
+2025-02-13 09:57:18.916 | INFO     | __main__:<module>:134 - remove batched parquets
+2025-02-13 09:57:18.917 | INFO     | __main__:<module>:137 - execution time: 0:08:34.366768
+2025-02-17 13:23:28.712 | INFO     | __main__:<module>:148 - Analysis date: 2025-01-15
+2025-02-17 13:25:45.034 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 0 to GCS: 0:02:16.307399
+2025-02-17 13:27:04.647 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 1 to GCS: 0:01:19.611117
+2025-02-17 13:31:26.029 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 2 to GCS: 0:04:21.381410
+2025-02-17 13:33:25.406 | INFO     | __main__:loop_through_batches_and_download_vp:111 - exported batch 3 to GCS: 0:01:59.351721
+2025-02-17 13:33:25.407 | INFO     | __main__:<module>:155 - execution time: 0:09:56.679670
+2025-02-17 13:33:49.071 | INFO     | __main__:<module>:97 - Analysis date: 2025-01-15
+2025-02-17 13:33:57.391 | INFO     | __main__:<module>:105 - concat and filter batched data: 0:00:08.319348
+2025-02-17 13:38:12.928 | INFO     | __main__:<module>:112 - export concatenated vp: 0:04:15.536791
+2025-02-17 13:42:21.135 | INFO     | __main__:<module>:134 - remove batched parquets
+2025-02-17 13:42:21.136 | INFO     | __main__:<module>:137 - execution time: 0:08:32.064286
diff --git a/gtfs_funnel/logs/vp_preprocessing.log b/gtfs_funnel/logs/vp_preprocessing.log
@@ -323,3 +323,13 @@
 2025-01-20 22:35:38.339 | INFO     | __main__:<module>:241 - merge with original and export: 0:01:29.188729
 2025-01-20 22:35:38.340 | INFO     | __main__:<module>:242 - vp with dwell time 2025-01-15: 0:05:49.716567
 2025-01-20 22:42:17.477 | INFO     | __main__:<module>:80 - 2025-01-15: condense vp for trip 0:06:20.345485
+2025-02-13 10:08:33.367 | INFO     | __main__:pare_down_to_valid_trips:88 - pare down vp: 0:02:19.071646
+2025-02-13 10:36:13.858 | INFO     | __main__:pare_down_to_valid_trips:88 - pare down vp: 0:02:43.933692
+2025-02-17 13:52:33.036 | INFO     | __main__:pare_down_to_valid_trips:88 - pare down vp: 0:02:23.300496
+2025-02-17 14:00:42.198 | INFO     | __main__:pare_down_to_valid_trips:94 - export vp direction: 0:08:09.162095
+2025-02-17 14:02:19.731 | INFO     | __main__:merge_in_vp_direction:143 - 2025-01-15: export usable vp with direction: 0:01:30.930961
+2025-02-17 14:02:19.974 | INFO     | __main__:<module>:215 - 2025-01-15: pare down vp, add direction execution time: 0:12:10.239378
+2025-02-17 14:07:29.338 | INFO     | __main__:<module>:216 - compute dwell df: 0:04:31.477602
+2025-02-17 14:08:50.431 | INFO     | __main__:<module>:241 - merge with original and export: 0:01:21.093084
+2025-02-17 14:08:50.432 | INFO     | __main__:<module>:242 - vp with dwell time 2025-01-15: 0:05:52.570686
+2025-02-17 14:15:23.550 | INFO     | __main__:<module>:80 - 2025-01-15: condense vp for trip 0:06:13.654816
diff --git a/high_quality_transit_areas/update_vars.py b/high_quality_transit_areas/update_vars.py
@@ -1,7 +1,7 @@
 from shared_utils import rt_dates
 import datetime as dt
 
-analysis_date = rt_dates.DATES["jan2025"]
+analysis_date = rt_dates.DATES["feb2025"]
 
 GCS_FILE_PATH = ("gs://calitp-analytics-data/data-analyses/"
                  "high_quality_transit_areas/")

diff --git a/open_data/update_vars.py b/open_data/update_vars.py
@@ -1,7 +1,7 @@
 from pathlib import Path
 from shared_utils import catalog_utils, rt_dates
 
-analysis_date = rt_dates.DATES["jan2025"]
+analysis_date = rt_dates.DATES["feb2025"]
 
 GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")
 

diff --git a/rt_scheduled_v_ran/scripts/update_vars.py b/rt_scheduled_v_ran/scripts/update_vars.py
@@ -7,7 +7,7 @@
 oct2024_week = rt_dates.get_week("oct2024", exclude_wed=True)
 
 
-analysis_date_list = [rt_dates.DATES["jan2025"]]
+analysis_date_list = [rt_dates.DATES["feb2025"]]
 # analysis_date_list = rt_dates.y2024_dates + rt_dates.y2023_dates
 
 GTFS_DATA_DICT = catalog_utils.get_catalog("gtfs_analytics_data")

diff --git a/rt_segment_speeds/logs/avg_speeds.log b/rt_segment_speeds/logs/avg_speeds.log
@@ -684,3 +684,9 @@
 2025-01-21 18:57:59.637 | INFO     | average_segment_speeds:segment_averages_detail:251 - speedmap_segments detailed segment averaging for ['2025-01-15'] execution time: 0:06:11.080021
 2025-01-21 19:03:28.422 | INFO     | average_segment_speeds:segment_averages:187 - speedmap_segments segment averaging for ['2025-01-15'] execution time: 0:05:28.685950
 2025-01-21 19:08:11.751 | INFO     | average_segment_speeds:segment_averages:187 - speedmap_segments segment averaging for ['2025-01-15'] execution time: 0:04:43.2400262025-02-12 10:11:30.775 | INFO     | __main__:segment_averages:187 - stop_segments segment averaging for ['2025-01-15'] execution time: 0:01:27.273686
+2025-02-17 16:14:49.090 | INFO     | __main__:segment_averages:187 - stop_segments segment averaging for ['2025-01-15'] execution time: 0:04:09.190074
+2025-02-17 17:06:27.731 | INFO     | __main__:summary_average_speeds:120 - trip avg 0:00:42.397680
+2025-02-17 17:06:41.442 | INFO     | __main__:summary_average_speeds:154 - rt_stop_times summary speed averaging for ['2025-01-15'] execution time: 0:00:56.108740
+2025-02-17 17:29:14.232 | INFO     | average_segment_speeds:segment_averages_detail:251 - speedmap_segments detailed segment averaging for ['2025-01-15'] execution time: 0:07:18.566542
+2025-02-17 17:35:34.968 | INFO     | average_segment_speeds:segment_averages:187 - speedmap_segments segment averaging for ['2025-01-15'] execution time: 0:06:20.571001
+2025-02-17 17:40:44.506 | INFO     | average_segment_speeds:segment_averages:187 - speedmap_segments segment averaging for ['2025-01-15'] execution time: 0:05:09.226820
diff --git a/rt_segment_speeds/logs/cut_stop_segments.log b/rt_segment_speeds/logs/cut_stop_segments.log
@@ -77,3 +77,5 @@
 2025-01-21 13:22:00.233 | INFO     | __main__:<module>:244 - speedmap segments and proxy_stop_times 2025-01-15: 0:05:55.448966
 2025-02-12 09:54:53.038 | INFO     | __main__:<module>:168 - cut segments 2025-01-15: 0:27:45.038713
 2025-02-12 10:02:11.788 | INFO     | __main__:<module>:244 - speedmap segments and proxy_stop_times 2025-01-15: 0:05:29.432920
+2025-02-17 15:13:39.338 | INFO     | __main__:<module>:168 - cut segments 2025-01-15: 0:24:38.593221
+2025-02-17 15:20:17.313 | INFO     | __main__:<module>:244 - speedmap segments and proxy_stop_times 2025-01-15: 0:05:03.685138
diff --git a/rt_segment_speeds/logs/interpolate_stop_arrival.log b/rt_segment_speeds/logs/interpolate_stop_arrival.log
@@ -141,3 +141,6 @@
 2025-01-21 18:35:30.275 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for rt_stop_times 2025-01-15:  2025-01-15: 0:13:06.847024
 2025-01-21 18:49:29.708 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-01-15:  2025-01-15: 0:02:15.045884
 2025-02-12 10:08:28.509 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for stop_segments 2025-01-15:  2025-01-15: 0:01:23.252567
+2025-02-17 16:08:39.466 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for stop_segments 2025-01-15:  2025-01-15: 0:09:31.159159
+2025-02-17 17:03:46.370 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for rt_stop_times 2025-01-15:  2025-01-15: 0:09:43.282306
+2025-02-17 17:18:36.944 | INFO     | interpolate_stop_arrival:interpolate_stop_arrivals:236 - interpolate arrivals for speedmap_segments 2025-01-15:  2025-01-15: 0:03:08.427741
diff --git a/rt_segment_speeds/logs/nearest_vp.log b/rt_segment_speeds/logs/nearest_vp.log
@@ -281,3 +281,6 @@
 2025-01-21 18:22:23.308 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for rt_stop_times 2025-01-15: 0:39:52.283432
 2025-01-21 18:47:14.614 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for speedmap_segments 2025-01-15: 0:08:43.206096
 2025-02-12 10:07:05.221 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for stop_segments 2025-01-15: 0:04:31.737901
+2025-02-17 15:59:08.244 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for stop_segments 2025-01-15: 0:38:30.266340
+2025-02-17 16:54:03.014 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for rt_stop_times 2025-01-15: 0:38:46.207664
+2025-02-17 17:15:28.470 | INFO     | nearest_vp_to_stop:nearest_neighbor_for_stop:188 - nearest neighbor for speedmap_segments 2025-01-15: 0:08:28.711118
diff --git a/rt_segment_speeds/logs/speeds_by_segment_trip.log b/rt_segment_speeds/logs/speeds_by_segment_trip.log
@@ -153,3 +153,6 @@
 2025-01-21 18:37:14.368 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for rt_stop_times 2025-01-15: 0:01:43.962507
 2025-01-21 18:51:30.158 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for speedmap_segments 2025-01-15: 0:01:50.399658
 2025-02-12 10:09:43.818 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for stop_segments 2025-01-15: 0:01:15.281024
+2025-02-17 16:10:20.211 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for stop_segments 2025-01-15: 0:01:40.637830
+2025-02-17 17:05:25.932 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for rt_stop_times 2025-01-15: 0:01:39.478984
+2025-02-17 17:21:33.315 | INFO     | stop_arrivals_to_speed:calculate_speed_from_stop_arrivals:176 - speeds by segment for speedmap_segments 2025-01-15: 0:02:45.108897
diff --git a/rt_segment_speeds/segment_speed_utils/project_vars.py b/rt_segment_speeds/segment_speed_utils/project_vars.py
@@ -11,7 +11,7 @@
 SHARED_GCS = GTFS_DATA_DICT.gcs_paths.SHARED_GCS
 PUBLIC_GCS = GTFS_DATA_DICT.gcs_paths.PUBLIC_GCS
 
-analysis_date = rt_dates.DATES["jan2025"]
+analysis_date = rt_dates.DATES["feb2025"]
 
 oct2023_week = rt_dates.get_week("oct2023", exclude_wed=True)
 apr2023_week = rt_dates.get_week("apr2023", exclude_wed=True)