diff --git a/dbt/models/staging/mastr/stg_mastr__storages.sql b/dbt/models/staging/mastr/stg_mastr__storages.sql index 19949c1..5ec536e 100644 --- a/dbt/models/staging/mastr/stg_mastr__storages.sql +++ b/dbt/models/staging/mastr/stg_mastr__storages.sql @@ -27,21 +27,36 @@ renamed_storage_units as ( renamed_extended as ( select + --general "EinheitMastrNummer" as mastr_id, "EinheitBetriebsstatus" as operating_status, - "Gemeindeschluessel" as municipality_id, - "Gemeinde" as municipality, - "Landkreis" as district, - "Postleitzahl" as zip_code, + "NetzbetreiberpruefungStatus" as grid_operator_inspection, + --dates + CASE + WHEN "Inbetriebnahmedatum" IS NOT NULL OR "GeplantesInbetriebnahmedatum" IS NOT NULL THEN + concat( + date_part('year', "Inbetriebnahmedatum"), + date_part('year', "GeplantesInbetriebnahmedatum") + )::integer + ELSE + NULL + END as installation_year, "Inbetriebnahmedatum" as commissioning_date, "GeplantesInbetriebnahmedatum" as planned_commissioning_date, - "Nettonennleistung" as power, "DatumDownload" as download_date, + --technical + "Nettonennleistung" as power_net, + "Bruttoleistung" as power_gross, + "ZugeordnenteWirkleistungWechselrichter" as power_inverter, + "Batterietechnologie" as battery_technology, + --owner + "AnlagenbetreiberMastrNummer" as unit_owner_mastr_id, + --location + "Gemeindeschluessel" as municipality_id, + "Gemeinde" as municipality, left("Gemeindeschluessel", 5) as district_id, - concat( - date_part('year', "Inbetriebnahmedatum"), - date_part('year', "GeplantesInbetriebnahmedatum") - ) as installation_year, + "Landkreis" as district, + "Postleitzahl" as zip_code, st_setsrid(st_point("Laengengrad", "Breitengrad"), 4326) as coordinate from source_extended @@ -50,25 +65,34 @@ renamed_extended as ( storage_units as ( select - renamed_storage_units.mastr_id as mastr_id, - renamed_extended.operating_status, - renamed_storage_units.storage_capacity, - renamed_extended.municipality_id, - renamed_extended.district_id, - renamed_extended.municipality, - renamed_extended.district, - renamed_extended.zip_code, - renamed_extended.commissioning_date, - renamed_extended.planned_commissioning_date, - renamed_extended.installation_year, - renamed_extended.power, - renamed_extended.download_date, - renamed_extended.coordinate - from renamed_storage_units - + --general + ru.mastr_id as mastr_id, + re.operating_status, + re.grid_operator_inspection, + --dates + re.installation_year, + re.commissioning_date, + re.planned_commissioning_date, + re.download_date, + --technical + ru.storage_capacity, + re.power_net, + re.power_gross, + re.power_inverter, + re.battery_technology, + --owner + re.unit_owner_mastr_id, + --location + re.municipality_id, + re.municipality, + re.district_id, + re.district, + re.zip_code, + re.coordinate + from renamed_extended as re left join - renamed_extended - on renamed_storage_units.mastr_id = renamed_extended.mastr_id + renamed_storage_units as ru + on re.mastr_id = ru.mastr_id ) select * from storage_units diff --git a/dbt/models/staging/mastr/stg_mastr__storages.yml b/dbt/models/staging/mastr/stg_mastr__storages.yml index 8be761d..0a4d3f1 100644 --- a/dbt/models/staging/mastr/stg_mastr__storages.yml +++ b/dbt/models/staging/mastr/stg_mastr__storages.yml @@ -5,25 +5,71 @@ models: columns: - name: mastr_id description: '' + data_type: character varying + tests: + - unique + - not_null: + config: + severity: warn # Don't understand why, but ~100 storages, mainly pumped hydro, dont have a mastr_id + - expect_column_values_to_match_regex: + name: regex_stg_mastr__storages_mastr_id + regex: "^[A-Z]{3}\\d{12}$" + - name: operating_status + data_type: character varying + description: '' + tests: + - not_null + - accepted_values: + name: accepted_values_stg_mastr__storages_operating_status + values: ['In Betrieb', 'In Planung', 'Endgültig stillgelegt', 'Vorübergehend stillgelegt'] + - name: grid_operator_inspection + data_type: character varying + description: '' + - name: installation_year + description: '' + data_type: integer + - name: commissioning_date + description: '' + data_type: date + - name: planned_commissioning_date + description: '' + data_type: date + - name: download_date + description: '' + data_type: date - name: storage_capacity description: '' - - name: municipality_id + data_type: double precision + - name: power_net + data_type: double precision description: '' - - name: district_id + - name: power_gross + data_type: double precision description: '' - - name: municipality + - name: power_inverter + data_type: double precision description: '' - - name: district + - name: battery_technology + data_type: character varying description: '' - - name: zip_code + - name: unit_owner_mastr_id + data_type: character varying description: '' - - name: commissioning_date + - name: municipality_id description: '' - - name: planned_commissioning_date + data_type: character varying + - name: municipality description: '' - - name: installation_year + data_type: character varying + - name: district_id + description: '' + data_type: text + - name: district description: '' - - name: power + data_type: character varying + - name: zip_code description: '' - - name: download_date + data_type: character varying + - name: coordinate + data_type: USER-DEFINED description: ''