Extend staging storages #9

OpenEnergyPlatform · Dec 5, 2023 · 54ae1f3 · 54ae1f3
1 parent 43f3e94
commit 54ae1f3
Show file tree

Hide file tree

Showing 2 changed files with 107 additions and 37 deletions.
diff --git a/dbt/models/staging/mastr/stg_mastr__storages.sql b/dbt/models/staging/mastr/stg_mastr__storages.sql
@@ -27,21 +27,36 @@ renamed_storage_units as (
 renamed_extended as (
 
     select
+        --general
         "EinheitMastrNummer" as mastr_id,
         "EinheitBetriebsstatus" as operating_status,
-        "Gemeindeschluessel" as municipality_id,
-        "Gemeinde" as municipality,
-        "Landkreis" as district,
-        "Postleitzahl" as zip_code,
+        "NetzbetreiberpruefungStatus" as grid_operator_inspection,
+        --dates
+        CASE
+        WHEN "Inbetriebnahmedatum" IS NOT NULL OR "GeplantesInbetriebnahmedatum" IS NOT NULL THEN
+            concat(
+            date_part('year', "Inbetriebnahmedatum"),
+            date_part('year', "GeplantesInbetriebnahmedatum")
+        )::integer
+        ELSE
+            NULL
+        END as installation_year,
         "Inbetriebnahmedatum" as commissioning_date,
         "GeplantesInbetriebnahmedatum" as planned_commissioning_date,
-        "Nettonennleistung" as power,
         "DatumDownload" as download_date,
+        --technical
+        "Nettonennleistung" as power_net,
+        "Bruttoleistung" as power_gross,
+        "ZugeordnenteWirkleistungWechselrichter" as power_inverter,
+        "Batterietechnologie" as battery_technology,
+        --owner
+        "AnlagenbetreiberMastrNummer" as unit_owner_mastr_id,
+        --location
+        "Gemeindeschluessel" as municipality_id,
+        "Gemeinde" as municipality,
         left("Gemeindeschluessel", 5) as district_id,
-        concat(
-            date_part('year', "Inbetriebnahmedatum"),
-            date_part('year', "GeplantesInbetriebnahmedatum")
-        ) as installation_year,
+        "Landkreis" as district,
+        "Postleitzahl" as zip_code,
         st_setsrid(st_point("Laengengrad", "Breitengrad"), 4326) as coordinate
     from source_extended
 
@@ -50,25 +65,34 @@ renamed_extended as (
 
 storage_units as (
     select
-        renamed_storage_units.mastr_id as mastr_id,
-        renamed_extended.operating_status,
-        renamed_storage_units.storage_capacity,
-        renamed_extended.municipality_id,
-        renamed_extended.district_id,
-        renamed_extended.municipality,
-        renamed_extended.district,
-        renamed_extended.zip_code,
-        renamed_extended.commissioning_date,
-        renamed_extended.planned_commissioning_date,
-        renamed_extended.installation_year,
-        renamed_extended.power,
-        renamed_extended.download_date,
-        renamed_extended.coordinate
-    from renamed_storage_units
-
+        --general
+        ru.mastr_id as mastr_id,
+        re.operating_status,
+        re.grid_operator_inspection,
+        --dates
+        re.installation_year,
+        re.commissioning_date,
+        re.planned_commissioning_date,
+        re.download_date,
+        --technical
+        ru.storage_capacity,
+        re.power_net,
+        re.power_gross,
+        re.power_inverter,
+        re.battery_technology,
+        --owner
+        re.unit_owner_mastr_id,
+        --location
+        re.municipality_id,
+        re.municipality,
+        re.district_id,
+        re.district,
+        re.zip_code,
+        re.coordinate
+    from renamed_extended as re
     left join
-        renamed_extended
-        on renamed_storage_units.mastr_id = renamed_extended.mastr_id
+        renamed_storage_units as ru
+        on re.mastr_id = ru.mastr_id
 )
 
 select * from storage_units
diff --git a/dbt/models/staging/mastr/stg_mastr__storages.yml b/dbt/models/staging/mastr/stg_mastr__storages.yml
@@ -5,25 +5,71 @@ models:
     columns:
       - name: mastr_id
         description: ''
+        data_type: character varying
+        tests:
+          - unique
+          - not_null:
+              config:
+                severity: warn # Don't understand why, but ~100 storages, mainly pumped hydro, dont have a mastr_id
+          - expect_column_values_to_match_regex:
+              name: regex_stg_mastr__storages_mastr_id
+              regex: "^[A-Z]{3}\\d{12}$"
+      - name: operating_status
+        data_type: character varying
+        description: ''
+        tests:
+          - not_null
+          - accepted_values:
+              name: accepted_values_stg_mastr__storages_operating_status
+              values: ['In Betrieb', 'In Planung', 'Endgültig stillgelegt', 'Vorübergehend stillgelegt']
+      - name: grid_operator_inspection
+        data_type: character varying
+        description: ''
+      - name: installation_year
+        description: ''
+        data_type: integer
+      - name: commissioning_date
+        description: ''
+        data_type: date
+      - name: planned_commissioning_date
+        description: ''
+        data_type: date
+      - name: download_date
+        description: ''
+        data_type: date
       - name: storage_capacity
         description: ''
-      - name: municipality_id
+        data_type: double precision
+      - name: power_net
+        data_type: double precision
         description: ''
-      - name: district_id
+      - name: power_gross
+        data_type: double precision
         description: ''
-      - name: municipality
+      - name: power_inverter
+        data_type: double precision
         description: ''
-      - name: district
+      - name: battery_technology
+        data_type: character varying
         description: ''
-      - name: zip_code
+      - name: unit_owner_mastr_id
+        data_type: character varying
         description: ''
-      - name: commissioning_date
+      - name: municipality_id
         description: ''
-      - name: planned_commissioning_date
+        data_type: character varying
+      - name: municipality
         description: ''
-      - name: installation_year
+        data_type: character varying
+      - name: district_id
+        description: ''
+        data_type: text
+      - name: district
         description: ''
-      - name: power
+        data_type: character varying
+      - name: zip_code
         description: ''
-      - name: download_date
+        data_type: character varying
+      - name: coordinate
+        data_type: USER-DEFINED
         description: ''