Merge pull request #1607 from cityofaustin/main

v2.4.1
cityofaustin · Nov 18, 2024 · b8485b2 · b8485b2
2 parents 457d064 + 877c65b
commit b8485b2
Show file tree

Hide file tree

Showing 11 changed files with 54 additions and 18 deletions.
diff --git a/database/migrations/default/1731348713148_moped_component_crashes_id_col/down.sql b/database/migrations/default/1731348713148_moped_component_crashes_id_col/down.sql
@@ -0,0 +1,2 @@
+alter table moped_component_crashes
+    drop column id;
diff --git a/database/migrations/default/1731348713148_moped_component_crashes_id_col/up.sql b/database/migrations/default/1731348713148_moped_component_crashes_id_col/up.sql
@@ -0,0 +1,4 @@
+alter table moped_component_crashes
+    add column id bigserial unique not null;
+
+create index on moped_component_crashes (id);
diff --git a/editor/package.json b/editor/package.json
@@ -1,6 +1,6 @@
 {
   "name": "vision-zero-editor",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "homepage": "./",
   "description": "ATD Vision Zero Editor",
   "author": "ATD Data & Technology Services",

diff --git a/etl/moped_projects/README.md b/etl/moped_projects/README.md
@@ -18,6 +18,6 @@ docker build . -t atddocker/vz-moped-join
 ```
 
 ```
-docker run -it --env-file env_file atddocker/vz-moped-join /bin/bash
+docker run -it --rm --network host --env-file env_file atddocker/vz-moped-join /bin/bash
 python moped_project_components_spatial_join.py
 ```
diff --git a/etl/moped_projects/env_template b/etl/moped_projects/env_template
@@ -3,5 +3,5 @@ MOPED_HASURA_ADMIN_SECRET=
 MOPED_HASURA_ENDPOINT=
 
 # Vision Zero Hasura
-VZ_HASURA_ADMIN_SECRET=
-VZ_HASURA_ENDPOINT=
+VZ_HASURA_ADMIN_SECRET=hasurapassword
+VZ_HASURA_ENDPOINT=http://localhost:8084/v1/graphql
diff --git a/etl/socrata_export/README.md b/etl/socrata_export/README.md
@@ -1,13 +1,14 @@
 # Open Data Portal (Socrata) Export
 
-This ETL downloads crash and people records from the VZ database and publishes them to the Open Data Portal, formerly known as Socrata.
+This ETL downloads crash and people records from the VZ database and publishes them to the Open Data Portal, formerly known as Socrata. It also publishes a crash <> moped project component lookup table.
 
-Records are pushed to two datasets, a crash-level dataset and a person-level dataset:
+Records are pushed to three datasets, a crash-level dataset, a person-level dataset, and a dataset that holds crash-moped relationships:
 
-| name    | dataset staging                                                                                                                         | dataset prod                                                                                                                                    |
-| ------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
-| crashes | [`3aut-fhzp`](https://datahub.austintexas.gov/Transportation-and-Mobility/Test-Crash-Report-Data/3aut-fhzp/about_data)                  | [`y2wy-tgr5`](https://datahub.austintexas.gov/Transportation-and-Mobility/Austin-Crash-Report-Data-Crash-Level-Records/y2wy-tgr5/about_data)    |
-| people  | [`v3x4-fjgm`](https://datahub.austintexas.gov/Transportation-and-Mobility/Test-Austin-Crash-Demographic-Statistics-incomplet/v3x4-fjgm) | [`xecs-rpy9`](https://data.austintexas.gov/Transportation-and-Mobility/Austin-Crash-Report-Data-Crash-Victim-Demographic-/xecs-rpy9/about_data) |
+| name             | dataset staging                                                                                                                         | dataset prod                                                                                                                                    |
+| ---------------- | --------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
+| crashes          | [`3aut-fhzp`](https://datahub.austintexas.gov/Transportation-and-Mobility/Test-Crash-Report-Data/3aut-fhzp/about_data)                  | [`y2wy-tgr5`](https://datahub.austintexas.gov/Transportation-and-Mobility/Austin-Crash-Report-Data-Crash-Level-Records/y2wy-tgr5/about_data)    |
+| people           | [`v3x4-fjgm`](https://datahub.austintexas.gov/Transportation-and-Mobility/Test-Austin-Crash-Demographic-Statistics-incomplet/v3x4-fjgm) | [`xecs-rpy9`](https://data.austintexas.gov/Transportation-and-Mobility/Austin-Crash-Report-Data-Crash-Victim-Demographic-/xecs-rpy9/about_data) |
+| crash-components | [`gb78-y2pd`](https://datahub.austintexas.gov/Transportation-and-Mobility/Vision-Zero-Crash-mobility-project-lookup-table/gb78-y2pd)    | [`usmd-qtb9`](https://datahub.austintexas.gov/Transportation-and-Mobility/Vision-Zero-Crash-mobility-project-lookup-table/usmd-qtb9/about_data) |
 
 ## Quick start
 
@@ -23,12 +24,12 @@ $ docker compose run socrata_export
 
 ```shell
 # from the socrata_export container's shell
-$ ./socrata_export.py --crashes --people
+$ ./socrata_export.py --crashes --people --crash-components
 ```
 
 ## CLI
 
-The CLI args `--crashes` and `--people` control which dataset(s) will be processed. At least one must be specified.
+The CLI args `--crashes` and `--people` and `--crash-components` control which dataset(s) will be processed. At least one must be specified.
 
 ```shell
 $ ./socrata_export.py --people

diff --git a/etl/socrata_export/env_template b/etl/socrata_export/env_template
@@ -6,3 +6,4 @@ SOCRATA_APP_TOKEN=
 # staging datasets
 SOCRATA_DATASET_CRASHES=3aut-fhzp
 SOCRATA_DATASET_PEOPLE=v3x4-fjgm
+SOCRATA_DATASET_CRASH_COMPONENTS=gb78-y2pd
diff --git a/etl/socrata_export/socrata_export.py b/etl/socrata_export/socrata_export.py
@@ -9,9 +9,10 @@
 
 SOCRATA_DATASET_CRASHES = os.getenv("SOCRATA_DATASET_CRASHES")
 SOCRATA_DATASET_PEOPLE = os.getenv("SOCRATA_DATASET_PEOPLE")
+SOCRATA_DATASET_CRASH_COMPONENTS = os.getenv("SOCRATA_DATASET_CRASH_COMPONENTS") or "gb78-y2pd"
 
 # number of records to download from Hasura and upload to Socrata
-RECORD_BATCH_SIZE = 1000
+DEFAULT_RECORD_BATCH_SIZE = 1000
 # number of times to try a single socrata upload
 MAX_UPLOAD_RETRIES = 1
 
@@ -21,12 +22,21 @@
         "typename": "socrata_export_crashes_view",
         "dataset_id": SOCRATA_DATASET_CRASHES,
         "query": QUERIES["crashes"],
+        "batch_size": DEFAULT_RECORD_BATCH_SIZE,
     },
     {
         "name": "people",
         "typename": "socrata_export_people_view",
         "dataset_id": SOCRATA_DATASET_PEOPLE,
         "query": QUERIES["people"],
+        "batch_size": DEFAULT_RECORD_BATCH_SIZE,
+    },
+    {
+        "name": "crash_components",
+        "typename": "moped_component_crashes",
+        "dataset_id": SOCRATA_DATASET_CRASH_COMPONENTS,
+        "query": QUERIES["crash_components"],
+        "batch_size": 10000,
     },
 ]
 
@@ -43,15 +53,16 @@ def main(args_dict):
 
         min_record_id_to_process = 0
         records_processed = 0
+        batch_size = dataset["batch_size"]
 
         while True:
             """
             Main loop for upserting records. we fetch and upload records based on the last
             minimum record ID that was processed. we continue until no records are returned
             from our Hasura query
             """
-            logger.info(f"Fetching up to {RECORD_BATCH_SIZE} records")
-            variables = {"limit": RECORD_BATCH_SIZE, "minId": min_record_id_to_process}
+            logger.info(f"Fetching up to {batch_size} records")
+            variables = {"limit": batch_size, "minId": min_record_id_to_process}
             records = make_hasura_request(query=dataset["query"], variables=variables)[
                 dataset["typename"]
             ]
@@ -102,9 +113,9 @@ def main(args_dict):
 if __name__ == "__main__":
     args = get_cli_args()
     logger = init_logger()
-    if not args.crashes and not args.people:
+    if not args.crashes and not args.people and not args.crash_components:
         raise ValueError(
-            "No dataset(s) specify. At least one of '--crashes' or '--people' is required"
+            "No dataset(s) specify. At least one '--crashes', '--people', or '--crash-components' is required"
         )
     args_dict = vars(args)
     main(args_dict)
diff --git a/etl/socrata_export/utils/cli.py b/etl/socrata_export/utils/cli.py
@@ -21,4 +21,9 @@ def get_cli_args():
         action="store_true",
         help="Export the crashes",
     )
+    parser.add_argument(
+        "--crash-components",
+        action="store_true",
+        help="Export the crash-components table",
+    )
     return parser.parse_args()
diff --git a/etl/socrata_export/utils/queries.py b/etl/socrata_export/utils/queries.py
@@ -71,7 +71,19 @@
 }
 """
 
+CRASH_COMPONENTS_QUERY = """
+query SocrataExportCrashComponents($limit: Int!, $minId: bigint!)  {
+  moped_component_crashes(order_by: {id: asc}, limit: $limit, where: {id: {_gte: $minId}}) {
+    id
+    crash_pk
+    mopd_proj_component_id
+  }
+}
+"""
+
+
 QUERIES = {
     "people": PEOPLE_QUERY,
     "crashes": CRASHES_QUERY,
+    "crash_components": CRASH_COMPONENTS_QUERY,
 }
diff --git a/viewer/package.json b/viewer/package.json
@@ -1,6 +1,6 @@
 {
   "name": "viewer",
-  "version": "2.4.0",
+  "version": "2.4.1",
   "homepage": "/viewer",
   "description": "Vision Zero Viewer",
   "author": "Austin Transportation & Public Works - Data & Technology Services",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		alter table moped_component_crashes
		drop column id;