Bring new NTD endpoint sources into the warehouse as staging

warehouse/models/staging/ntd_annual_data_tables/2022/_src.yml

@@ -0,0 +1,42 @@
+version: 2
+
+sources:
+  - name: external_ntd__annual_reporting
+    description: Annual data tables for the year 2022, loaded from DOT NTD API https://www.transit.dot.gov/ntd/ntd-data.
+    database: "{{ env_var('DBT_SOURCE_DATABASE', var('SOURCE_DATABASE')) }}"
+    schema: external_ntd__annual_reporting
+    tables:
+      - name: 2022__stations_and_facilities_by_agency_and_facility_type
+      - name: 2022__stations_by_mode_and_age
+      - name: 2022__operating_expenses_by_function
+      - name: 2022__operating_expenses_by_function_and_agency
+      - name: 2022__operating_expenses_by_type
+      - name: 2022__operating_expenses_by_type_and_agency
+      - name: 2022__vehicles_age_distribution
+      - name: 2022__vehicles_type_count_by_agency
+      - name: 2022__service_by_mode_and_time_period
+      - name: 2022__service_by_mode
+      - name: 2022__service_by_agency
+      - name: 2022__track_and_roadway_by_mode
+      - name: 2022__track_and_roadway_guideway_age_distribution
+      - name: 2022__track_and_roadway_by_agency
+      - name: 2022__metrics
+      - name: 2022__funding_sources_by_expense_type
+      - name: 2022__funding_sources_directly_generated
+      - name: 2022__funding_sources_taxes_levied_by_agency
+      - name: 2022__funding_sources_local
+      - name: 2022__funding_sources_state
+      - name: 2022__funding_sources_federal
+      - name: 2022__maintenance_facilities
+      - name: 2022__maintenance_facilities_by_agency
+      - name: 2022__fuel_and_energy
+      - name: 2022__fuel_and_energy_by_agency
+      - name: 2022__employees_by_mode_and_employee_type
+      - name: 2022__employees_by_mode
+      - name: 2022__employees_by_agency
+      - name: 2022__capital_expenses_by_capital_use
+      - name: 2022__capital_expenses_by_mode
+      - name: 2022__capital_expenses_for_existing_service
+      - name: 2022__capital_expenses_for_expansion_of_service
+      - name: 2022__breakdowns
+      - name: 2022__breakdowns_by_agency

warehouse/models/staging/ntd_annual_data_tables/2022/_stg_ntd_annual_data_tables.yml

@@ -0,0 +1,37 @@
+version: 2
+
+models:
+  - name: stg_ntd_annual_data__2022__stations_and_facilities_by_agency_and_facility_type
+  - name: stg_ntd_annual_data__2022__stations_by_mode_and_age
+  - name: stg_ntd_annual_data__2022__operating_expenses_by_function
+  - name: stg_ntd_annual_data__2022__operating_expenses_by_function_and_agency
+  - name: stg_ntd_annual_data__2022__operating_expenses_by_type
+  - name: stg_ntd_annual_data__2022__operating_expenses_by_type_and_agency
+  - name: stg_ntd_annual_data__2022__vehicles_age_distribution
+  - name: stg_ntd_annual_data__2022__vehicles_type_count_by_agency
+  - name: stg_ntd_annual_data__2022__service_by_mode_and_time_period
+  - name: stg_ntd_annual_data__2022__service_by_mode
+  - name: stg_ntd_annual_data__2022__service_by_agency
+  - name: stg_ntd_annual_data__2022__track_and_roadway_by_mode
+  - name: stg_ntd_annual_data__2022__track_and_roadway_guideway_age_distribution
+  - name: stg_ntd_annual_data__2022__track_and_roadway_by_agency
+  - name: stg_ntd_annual_data__2022__metrics
+  - name: stg_ntd_annual_data__2022__funding_sources_by_expense_type
+  - name: stg_ntd_annual_data__2022__funding_sources_directly_generated
+  - name: stg_ntd_annual_data__2022__funding_sources_taxes_levied_by_agency
+  - name: stg_ntd_annual_data__2022__funding_sources_local
+  - name: stg_ntd_annual_data__2022__funding_sources_state
+  - name: stg_ntd_annual_data__2022__funding_sources_federal
+  - name: stg_ntd_annual_data__2022__maintenance_facilities
+  - name: stg_ntd_annual_data__2022__maintenance_facilities_by_agency
+  - name: stg_ntd_annual_data__2022__fuel_and_energy
+  - name: stg_ntd_annual_data__2022__fuel_and_energy_by_agency
+  - name: stg_ntd_annual_data__2022__employees_by_mode_and_employee_type
+  - name: stg_ntd_annual_data__2022__employees_by_mode
+  - name: stg_ntd_annual_data__2022__employees_by_agency
+  - name: stg_ntd_annual_data__2022__capital_expenses_by_capital_use
+  - name: stg_ntd_annual_data__2022__capital_expenses_by_mode
+  - name: stg_ntd_annual_data__2022__capital_expenses_for_existing_service
+  - name: stg_ntd_annual_data__2022__capital_expenses_for_expansion_of_service
+  - name: stg_ntd_annual_data__2022__breakdowns
+  - name: stg_ntd_annual_data__2022__breakdowns_by_agency

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__breakdowns.sql

@@ -0,0 +1,19 @@
+WITH external_2022_breakdowns AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__breakdowns') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_breakdowns
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__breakdowns AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__breakdowns

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__breakdowns_by_agency.sql

@@ -0,0 +1,19 @@
+WITH external_2022_breakdowns_by_agency AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__breakdowns_by_agency') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_breakdowns_by_agency
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__breakdowns_by_agency AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__breakdowns_by_agency

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__capital_expenses_by_capital_use.sql

@@ -0,0 +1,19 @@
+WITH external_2022_capital_expenses_by_capital_use AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__capital_expenses_by_capital_use') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_capital_expenses_by_capital_use
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__capital_expenses_by_capital_use AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__capital_expenses_by_capital_use

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__capital_expenses_by_mode.sql

@@ -0,0 +1,19 @@
+WITH external_2022_capital_expenses_by_mode AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__capital_expenses_by_mode') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_capital_expenses_by_mode
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__capital_expenses_by_mode AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__capital_expenses_by_mode

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__capital_expenses_for_existing_service.sql

@@ -0,0 +1,19 @@
+WITH external_2022_capital_expenses_for_existing_service AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__capital_expenses_for_existing_service') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_capital_expenses_for_existing_service
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__capital_expenses_for_existing_service AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__capital_expenses_for_existing_service

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__capital_expenses_for_expansion_of_service.sql

@@ -0,0 +1,19 @@
+WITH external_2022_capital_expenses_for_expansion_of_service AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__capital_expenses_for_expansion_of_service') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_capital_expenses_for_expansion_of_service
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__capital_expenses_for_expansion_of_service AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__capital_expenses_for_expansion_of_service

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__employees_by_agency.sql

@@ -0,0 +1,19 @@
+WITH external_2022_employees_by_agency AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__employees_by_agency') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_employees_by_agency
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__employees_by_agency AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__employees_by_agency

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__employees_by_mode.sql

@@ -0,0 +1,19 @@
+WITH external_2022_employees_by_mode AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__employees_by_mode') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_employees_by_mode
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__employees_by_mode AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__employees_by_mode

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__employees_by_mode_and_employee_type.sql

@@ -0,0 +1,19 @@
+WITH external_2022_employees_by_mode_and_employee_type AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__employees_by_mode_and_employee_type') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_employees_by_mode_and_employee_type
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__employees_by_mode_and_employee_type AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__employees_by_mode_and_employee_type

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__fuel_and_energy.sql

@@ -0,0 +1,19 @@
+WITH external_2022_fuel_and_energy AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__fuel_and_energy') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_fuel_and_energy
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__fuel_and_energy AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__fuel_and_energy

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__fuel_and_energy_by_agency.sql

@@ -0,0 +1,19 @@
+WITH external_2022_fuel_and_energy_by_agency AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__fuel_and_energy_by_agency') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_fuel_and_energy_by_agency
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__fuel_and_energy_by_agency AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__fuel_and_energy_by_agency

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__funding_sources_by_expense_type.sql

@@ -0,0 +1,19 @@
+WITH external_2022_funding_sources_by_expense_type AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__funding_sources_by_expense_type') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_funding_sources_by_expense_type
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__funding_sources_by_expense_type AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__funding_sources_by_expense_type

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__funding_sources_directly_generated.sql

@@ -0,0 +1,19 @@
+WITH external_2022_funding_sources_directly_generated AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__funding_sources_directly_generated') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_funding_sources_directly_generated
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__funding_sources_directly_generated AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__funding_sources_directly_generated

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__funding_sources_federal.sql

@@ -0,0 +1,19 @@
+WITH external_2022_funding_sources_federal AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__funding_sources_federal') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_funding_sources_federal
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__funding_sources_federal AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__funding_sources_federal

warehouse/models/staging/ntd_annual_data_tables/2022/stg_ntd_annual_data__2022__funding_sources_local.sql

@@ -0,0 +1,19 @@
+WITH external_2022_funding_sources_local AS (
+    SELECT *
+    FROM {{ source('external_ntd__annual_reporting', '2022__funding_sources_local') }}
+),
+
+get_latest_extract AS(
+
+    SELECT *
+    FROM external_2022_funding_sources_local
+    -- we pull the whole table every month in the pipeline, so this gets only the latest extract
+    QUALIFY DENSE_RANK() OVER (ORDER BY execution_ts DESC) = 1
+),
+
+stg_ntd_annual_data__2022__funding_sources_local AS (
+    SELECT *
+    FROM get_latest_extract
+)
+
+SELECT * FROM stg_ntd_annual_data__2022__funding_sources_local