DCV-2934 changes to setup task

copier-answers.yml

@@ -5,3 +5,5 @@ use_dbt_checkpoint: true
 use_sqlfluff: true
 use_yamllint: true
 yml_dags_path: orchestrate/dag_yml_definitions
+setup_dbtcoves_config: true
+setup_dbtcoves_templates: true

copier.yml

@@ -43,6 +43,24 @@ use_yamllint:
   when: "{{setup_precommit}}"
   help: use Yamllint?
 
+add_dbtcoves_config_and_templates:
+  type: bool
+  default: false
+  when: "{{not setup_dbt_coves and setup_dbt_project}}"
+  help: Add dbt-coves config and templates?
+
+setup_dbtcoves_config:
+  type: bool
+  default: false
+  when: "{{setup_dbt_coves}}"
+  help: Setup dbt-coves config?
+
+setup_dbtcoves_templates:
+  type: bool
+  default: false
+  when: "{{setup_dbt_coves}}"
+  help: Setup dbt-coves templates?
+
 dbt_adapter:
   type: str
   choices:

{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_config or add_dbtcoves_config_and_templates %}config.yml{% endif %}

@@ -0,0 +1,62 @@
+generate:
+  sources:
+    database: RAW # Database where to look for source tables
+    sources_destination: "models/L1_staging/{{schema}}/_{{schema}}.yml" # Where sources yml files will be generated
+    models_destination: "models/L1_staging/{{schema}}/{{relation}}.sql" # Where models sql files will be generated
+    model_props_destination: "models/L1_staging/{{schema}}/{{relation}}.yml" # Where models yml files will be generated
+    update_strategy: update # Action to perform when a property file exists. Options: update, recreate, fail, ask
+    templates_folder: ".dbt_coves/templates" # Folder where source generation jinja templates are located.
+    flatten_json_fields: "no" # Action to perform when VARIANT / JSON field is encountered
+
+  properties:
+    destination: "{{model_folder_path}}/{{model_file_name}}.yml" # Where models yml files will be generated
+    # You can specify a different path by declaring it explicitly, i.e.: "models/staging/{{model_file_name}}.yml"
+    update_strategy: ask # Action to perform when a property file already exists. Options: update, recreate, fail, ask
+    models: "models/" # Model(s) path where 'generate properties' will look for models for generation
+
+  metadata:
+    database: RAW
+    # destination: "metadata.csv"
+
+  airflow_dags:
+    yml_path: "/config/workspace/{{ env_var('DATACOVES__AIRFLOW_DAGS_YML_PATH') }}"
+    dags_path: "/config/workspace/{{ env_var('DATACOVES__AIRFLOW_DAGS_PATH') }}"
+
+    # UNCOMMENT THE FOLLOWING LINES TO ENABLE AIRFLOW DAGS GENERATION
+    # BASED ON AIRBYTE AND FIVETRAN CONNECTIONS
+    # generators_params:
+    #   AirbyteDbtGenerator:
+    #     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+    #     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
+    #     airbyte_conn_id: airbyte_connection
+
+    #     dbt_project_path: "{{ env_var('DATACOVES__DBT_HOME') }}"
+    #     run_dbt_compile: false
+    #     run_dbt_deps: false
+
+    #   AirbyteGenerator:
+    #     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+    #     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
+    #     airbyte_conn_id: airbyte_connection
+
+    #   FivetranDbtGenerator:
+    #     api_key: "{{ env_var('DATACOVES__FIVETRAN_API_KEY') }}"
+    #     api_secret: "{{ env_var('DATACOVES__FIVETRAN_API_SECRET') }}"
+
+    #     dbt_project_path: "{{ env_var('DATACOVES__DBT_HOME') }}"
+    #     run_dbt_compile: false
+    #     run_dbt_deps: false
+
+    #   FivetranGenerator:
+    #     api_key: "{{ env_var('DATACOVES__FIVETRAN_API_KEY') }}"
+    #     api_secret: "{{ env_var('DATACOVES__FIVETRAN_API_SECRET') }}"
+    #     connection_ids:
+    #       - fivetran-connection-id-1
+    #       - fivetran-connection-id-2
+
+# UNCOMMENT THE FOLLOWING LINES TO ENABLE AIRBYTE EXTRACTION
+# extract:
+#   airbyte:
+#     # path: /config/workspace/load/airbyte
+#     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+#     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"

{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates or add_dbtcoves_config_and_templates %}templates{% endif %}/model_props.yml

@@ -0,0 +1,12 @@
+version: 2
+
+models:
+  - name: {{ model | lower }}
+    description: ''
+    columns:
+{%- for col in columns %}
+      - name: {{ col['id'] }}
+      {%- if col['description'] %}
+        description: "{{ col['description'] }}"
+      {%- endif %}
+{%- endfor %}

{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates or add_dbtcoves_config_and_templates %}templates{% endif %}/source_props.yml

@@ -0,0 +1,10 @@
+version: 2
+
+sources:
+  - name: {{ relation.schema }}
+{%- if source_database %}
+    database: {{ source_database }}
+{%- endif %}
+    tables:
+      - name: {{ relation.name }}
+        description: ''

{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates or add_dbtcoves_config_and_templates %}templates{% endif %}/staging_model.sql

@@ -0,0 +1,35 @@
+with raw_source as (
+
+    select *
+    from {% raw %}{{{% endraw %} source('{{ relation.schema }}', '{{ relation.name }}') {% raw %}}}{% endraw %}
+
+),
+
+final as (
+
+    select
+{%- if adapter_name == 'SnowflakeAdapter' or adapter_name == 'RedshiftAdapter' %}
+{%- for key, cols in nested.items() %}
+  {%- for col in cols %}
+        {{ key.lower() }}:{{ '"' + col + '"' }}::{{ cols[col]["type"].lower() }} as {{ cols[col]["id"] }}{% if not loop.last or columns %},{% endif %}
+  {%- endfor %}
+{%- endfor %}
+{%- for col in columns %}
+        {{ '"' + col['name'] + '"' }}::{{ col["type"].lower() }} as {{ col['id'] }}{% if not loop.last %},{% endif %}
+{%- endfor %}
+{%- elif adapter_name == 'BigQueryAdapter' %}
+{%- for key, cols in nested.items() %}
+  {%- for col in cols %}
+        cast({{ key }}.{{ col }} as {{ cols[col]["type"].lower().replace("varchar", "string") }}) as {{ cols[col]["id"] }}{% if not loop.last or columns %},{% endif %}
+  {%- endfor %}
+{%- endfor %}
+{%- for col in columns %}
+        cast({{ col['name'] }} as {{ col["type"].lower().replace("varchar", "string") }}) as {{ col['id'] }}{% if not loop.last %},{% endif %}
+{%- endfor %}
+{%- endif %}
+
+    from raw_source
+
+)
+
+select * from final

{% if setup_dbt_coves or add_dbtcoves_config_and_templates %}{{ dbt_project_dir }}{% endif %}/.dbt_coves/{% if setup_dbtcoves_templates or add_dbtcoves_config_and_templates %}templates{% endif %}/staging_model_props.yml

@@ -0,0 +1,20 @@
+version: 2
+
+models:
+  - name: {{model}}
+    description: ''
+    columns:
+{%- for cols in nested.values() %}
+  {%- for col in cols %}
+      - name: {{ cols[col]["id"] }}
+      {%- if cols[col]["description"] %}
+        description: "{{ cols[col]['description'] }}"
+      {%- endif %}
+  {%- endfor %}
+{%- endfor %}
+{%- for col in columns %}
+      - name: {{ col['id'] }}
+      {%- if col['description'] %}
+        description: "{{ col['description'] }}"
+      {%- endif %}
+{%- endfor %}