add new external table for 2022 agency info, fix bugs in previous ext…

…ernal table creation
cal-itp · Dec 18, 2024 · 3404297 · 3404297
1 parent 43928ce
commit 3404297
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 2 deletions.
diff --git a/...ags/create_external_tables/ntd_data_products/2022__annual_database_agency_information.yml b/...ags/create_external_tables/ntd_data_products/2022__annual_database_agency_information.yml
@@ -0,0 +1,16 @@
+operator: operators.ExternalTable
+bucket: gs://calitp-ntd-xlsx-products-clean
+prefix_bucket: false
+post_hook: |
+  SELECT *
+  FROM `{{ get_project_id() }}`.external_ntd__annual_reporting.2022__annual_database_agency_information
+  LIMIT 1;
+source_objects:
+  - "annual_database_agency_information/2022/_2022_agency_information/*.jsonl.gz"
+destination_project_dataset_table: "external_ntd__annual_reporting.2022__annual_database_agency_information"
+source_format: NEWLINE_DELIMITED_JSON
+use_bq_client: true
+hive_options:
+  mode: AUTO
+  require_partition_filter: false
+  source_uri_prefix: "annual_database_agency_information/2022/_2022_agency_information/"
diff --git a/...ags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml b/...ags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml
@@ -1,5 +1,5 @@
 operator: operators.ExternalTable
-bucket: gs://test-calitp-ntd-xlsx-products-clean
+bucket: gs://calitp-ntd-xlsx-products-clean
 prefix_bucket: false
 post_hook: |
   SELECT *

diff --git a/...ate_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml b/...ate_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml
@@ -1,5 +1,5 @@
 operator: operators.ExternalTable
-bucket: gs://test-calitp-ntd-xlsx-products-clean
+bucket: gs://calitp-ntd-xlsx-products-clean
 prefix_bucket: false
 post_hook: |
   SELECT *