diff --git a/airflow/dags/create_external_tables/ntd_data_products/2022__annual_database_agency_information.yml b/airflow/dags/create_external_tables/ntd_data_products/2022__annual_database_agency_information.yml new file mode 100644 index 0000000000..88aa5add97 --- /dev/null +++ b/airflow/dags/create_external_tables/ntd_data_products/2022__annual_database_agency_information.yml @@ -0,0 +1,16 @@ +operator: operators.ExternalTable +bucket: gs://calitp-ntd-xlsx-products-clean +prefix_bucket: false +post_hook: | + SELECT * + FROM `{{ get_project_id() }}`.external_ntd__annual_reporting.2022__annual_database_agency_information + LIMIT 1; +source_objects: + - "annual_database_agency_information/2022/_2022_agency_information/*.jsonl.gz" +destination_project_dataset_table: "external_ntd__annual_reporting.2022__annual_database_agency_information" +source_format: NEWLINE_DELIMITED_JSON +use_bq_client: true +hive_options: + mode: AUTO + require_partition_filter: false + source_uri_prefix: "annual_database_agency_information/2022/_2022_agency_information/" diff --git a/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml b/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml index e17a51ac6d..be1db0b31a 100644 --- a/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml +++ b/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_agency_information.yml @@ -1,5 +1,5 @@ operator: operators.ExternalTable -bucket: gs://test-calitp-ntd-xlsx-products-clean +bucket: gs://calitp-ntd-xlsx-products-clean prefix_bucket: false post_hook: | SELECT * diff --git a/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml b/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml index 6b7760b716..58a521ac87 100644 --- a/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml +++ b/airflow/dags/create_external_tables/ntd_data_products/2023__annual_database_contractual_relationships.yml @@ -1,5 +1,5 @@ operator: operators.ExternalTable -bucket: gs://test-calitp-ntd-xlsx-products-clean +bucket: gs://calitp-ntd-xlsx-products-clean prefix_bucket: false post_hook: | SELECT *