From d6be78d28bda50844f5e53017194b3c702a96c3c Mon Sep 17 00:00:00 2001 From: Colin Dugeai <46027833+cdugeai@users.noreply.github.com> Date: Mon, 4 Dec 2023 17:39:14 +0100 Subject: [PATCH] Fixes df3 (#673) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix directeurs_administration_centrale on corrige directeurs_administration_centrale fix directeurs_administration_centrale (2) * Cohabitation df1-df3 + fix df3_chantier (#671) * dbt.model.new: df1_indic et df1_ch Ajout de ces models pour changer facilement les tables d'exposition (indicateur et chantier) * dbt.model.new: df1_indic et df1_ch Ajout de ces models pour changer facilement les tables d'exposition (indicateur et chantier) * fix empty array * fix: directions_administration_centrale on récupère correctement les directions_administration_centrale à l'aide de la table metadata_porteur et non plus dans la colonne porteur_shorts_DAC de metadata_chantier qui était présente pour information donc sur laquelle il ne faut pas se baser. * replace dy dbt ref * dbt.model.new: df1_indic et df1_ch Ajout de ces models pour changer facilement les tables d'exposition (indicateur et chantier) * fix empty array * fix: directions_administration_centrale on récupère correctement les directions_administration_centrale à l'aide de la table metadata_porteur et non plus dans la colonne porteur_shorts_DAC de metadata_chantier qui était présente pour information donc sur laquelle il ne faut pas se baser. * replace dy dbt ref * update dbt tests * fix tests issue: https://github.com/dbt-labs/dbt-core/issues/3229 solution: given by dbt cli when error * hotfix: Revert "feat: ajout de la météo dans import massif de commentaires (000)" This reverts commit 526bfc62a1fd4a37f00922fd23812966a317ae4d. * compare ta_ch et ta_indic * compute_ta_indic: add 2 rules 2 rules: - si hausse et va>=vc -> 100% - si baisse et va<=vc -> 100% * fix: handle ta=nouveau_ta then ratio will be 0 * update dump scripts * Update .gitignore * Update README.md * update migration process --- .../macros/df3_validation/df3_date_early.sql | 2 +- .../macros/df3_validation/df3_date_late.sql | 2 +- .../macros/df3_validation/df3_not_equal.sql | 2 +- .../df3_validation/df3_not_equal_bool.sql | 2 +- .../df3_validation/df3_should_notnull.sql | 2 +- .../macros/df3_validation/df3_should_null.sql | 2 +- .../macros/df3_validation/schema.yml | 24 +-- .../models/analysis/compare_ta_chantier.sql | 41 +++++ .../models/analysis/compare_ta_indic.sql | 41 +++++ .../models/df3/4_compute/compute_ta_indic.sql | 19 ++- .../models/df3/5_exposition/df3_chantier.sql | 24 ++- .../models/exposition/chantier.sql | 148 +----------------- .../models/exposition/df1_chantier.sql | 147 +++++++++++++++++ .../models/exposition/df1_indicateur.sql | 51 ++++++ .../models/exposition/indicateur.sql | 54 +------ .../data_factory/models/exposition/schema.yml | 6 + .../df3/a_suppr/df3_asuppr_different.sql | 1 + .../a_suppr/df3_asuppr_should_not_null.sql | 1 + .../df3/a_suppr/df3_asuppr_should_null.sql | 1 + .../tests/df3/baro/df3_baro_different.sql | 1 + .../df3/baro/df3_baro_should_not_null.sql | 1 + .../tests/df3/baro/df3_baro_should_null.sql | 1 + .../tests/df3/ta/df3_ta_different.sql | 1 + .../tests/df3/ta/df3_ta_should_not_null.sql | 1 + .../tests/df3/ta/df3_ta_should_null.sql | 1 + .../tests/df3/va/df3_va_date_late.sql | 1 + .../df3/va/df3_va_date_should_not_null.sql | 1 + .../tests/df3/va/df3_va_date_should_null.sql | 1 + .../tests/df3/va/df3_va_date_soon.sql | 1 + .../tests/df3/va/df3_va_different.sql | 1 + .../tests/df3/va/df3_va_should_not_null.sql | 1 + .../tests/df3/va/df3_va_should_null.sql | 1 + .../df3/va_unnest/df3_vaunnest_different.sql | 1 + .../df3/va_unnest/df3_vaunnest_missing.sql | 1 + .../df3/va_unnest/df3_vaunnest_suppl.sql | 1 + .../tests/df3/vca/df3_vca_date_late.sql | 1 + .../df3/vca/df3_vca_date_should_not_null.sql | 1 + .../df3/vca/df3_vca_date_should_null.sql | 1 + .../tests/df3/vca/df3_vca_date_soon.sql | 1 + .../tests/df3/vca/df3_vca_different.sql | 1 + .../tests/df3/vca/df3_vca_should_not_null.sql | 1 + .../tests/df3/vca/df3_vca_should_null.sql | 1 + .../tests/df3/vcg/df3_vcg_date_late.sql | 1 + .../df3/vcg/df3_vcg_date_should_not_null.sql | 1 + .../df3/vcg/df3_vcg_date_should_null.sql | 1 + .../tests/df3/vcg/df3_vcg_date_soon.sql | 1 + .../tests/df3/vcg/df3_vcg_different.sql | 1 + .../tests/df3/vcg/df3_vcg_should_not_null.sql | 1 + .../tests/df3/vcg/df3_vcg_should_null.sql | 1 + .../tests/df3/vi/df3_vi_date_late.sql | 1 + .../df3/vi/df3_vi_date_should_not_null.sql | 1 + .../tests/df3/vi/df3_vi_date_should_null.sql | 1 + .../tests/df3/vi/df3_vi_date_soon.sql | 1 + .../tests/df3/vi/df3_vi_different.sql | 1 + .../tests/df3/vi/df3_vi_should_not_null.sql | 1 + .../tests/df3/vi/df3_vi_should_null.sql | 1 + .../tests/df3/z_appl/df3_zappl_different.sql | 1 + .../df3/z_appl/df3_zappl_should_not_null.sql | 1 + .../df3/z_appl/df3_zappl_should_null.sql | 1 + data_management/scripts/migration/README.md | 2 +- .../scripts/migration/dump_prod.sh | 15 +- .../scripts/migration/load_to_local.sh | 4 +- .../scripts/migration/out/.gitignore | 5 +- 63 files changed, 411 insertions(+), 225 deletions(-) create mode 100644 data_management/data_factory/models/analysis/compare_ta_chantier.sql create mode 100644 data_management/data_factory/models/analysis/compare_ta_indic.sql create mode 100644 data_management/data_factory/models/exposition/df1_chantier.sql create mode 100644 data_management/data_factory/models/exposition/df1_indicateur.sql diff --git a/data_management/data_factory/macros/df3_validation/df3_date_early.sql b/data_management/data_factory/macros/df3_validation/df3_date_early.sql index 06e49f1f6..f83aa6513 100644 --- a/data_management/data_factory/macros/df3_validation/df3_date_early.sql +++ b/data_management/data_factory/macros/df3_validation/df3_date_early.sql @@ -20,7 +20,7 @@ with jjoin as (select else false end as ignore_row from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/df3_date_late.sql b/data_management/data_factory/macros/df3_validation/df3_date_late.sql index a355cf3a4..a41168523 100644 --- a/data_management/data_factory/macros/df3_validation/df3_date_late.sql +++ b/data_management/data_factory/macros/df3_validation/df3_date_late.sql @@ -20,7 +20,7 @@ with jjoin as (select else false end as ignore_row from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/df3_not_equal.sql b/data_management/data_factory/macros/df3_validation/df3_not_equal.sql index 50880a01b..4978c692a 100644 --- a/data_management/data_factory/macros/df3_validation/df3_not_equal.sql +++ b/data_management/data_factory/macros/df3_validation/df3_not_equal.sql @@ -19,7 +19,7 @@ with jjoin as (select end as ratio from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/df3_not_equal_bool.sql b/data_management/data_factory/macros/df3_validation/df3_not_equal_bool.sql index 4fa76bba3..2cc285482 100644 --- a/data_management/data_factory/macros/df3_validation/df3_not_equal_bool.sql +++ b/data_management/data_factory/macros/df3_validation/df3_not_equal_bool.sql @@ -15,7 +15,7 @@ with jjoin as (select null::numeric as ratio from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/df3_should_notnull.sql b/data_management/data_factory/macros/df3_validation/df3_should_notnull.sql index 26d1dfcdf..0023ce016 100644 --- a/data_management/data_factory/macros/df3_validation/df3_should_notnull.sql +++ b/data_management/data_factory/macros/df3_validation/df3_should_notnull.sql @@ -13,7 +13,7 @@ with jjoin as (select a.{{colname}} as computed_value, b.{{colname}} as target_value from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/df3_should_null.sql b/data_management/data_factory/macros/df3_validation/df3_should_null.sql index b5fa355b1..99e2152cb 100644 --- a/data_management/data_factory/macros/df3_validation/df3_should_null.sql +++ b/data_management/data_factory/macros/df3_validation/df3_should_null.sql @@ -13,7 +13,7 @@ with jjoin as (select a.{{colname}} as computed_value, b.{{colname}} as target_value from {{ ref('df3_indicateur') }} a - full join {{ ref('indicateur') }} b + full join {{ ref('df1_indicateur') }} b on a.territoire_code=b.territoire_code and a.id=b.id ) diff --git a/data_management/data_factory/macros/df3_validation/schema.yml b/data_management/data_factory/macros/df3_validation/schema.yml index f15936203..87d293757 100644 --- a/data_management/data_factory/macros/df3_validation/schema.yml +++ b/data_management/data_factory/macros/df3_validation/schema.yml @@ -3,64 +3,64 @@ version: 2 macros: - name: df3_date_early description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Compare si une date est **antérieure** à ce qui est attendu. arguments: - name: colname_date type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant une date à vérifier. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant une date à vérifier. - name: ignore_day type: boolean description: Indique si on doit ignorer le jour du mois. Ie, si le mois est identique, on considère les dates identiques. - name: df3_date_late description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Compare si une date est **postérieure** à ce qui est attendu. arguments: - name: colname_date type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant une date à vérifier. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant une date à vérifier. - name: ignore_day type: boolean description: Indique si on doit ignorer le jour du mois. Ie, si le mois est identique, on considère les dates identiques. - name: df3_not_equal_bool description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Compare si deux booléens sont **non null et identiques**. arguments: - name: colname type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant un booléen à comparer. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant un booléen à comparer. - name: df3_not_equal description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Compare si deux valeurs numériques sont égales. Le paramètre `tolerance` permet d'indiquer une fenêtre de précision. arguments: - name: colname type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant un nombre à comparer. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant un nombre à comparer. - name: tolerance type: float description: Nombre dans `[0,1]` qui indique le seuil de pourcentage de différence des deux nombres au dessus duquel on considère que les deux nombres sont **différents**. Par exemple, `0.01` signifie que deux nombres seront considérés non égaux s'ils varient de `1%` ou plus. - name: df3_should_notnull description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Indique lorsqu'une valeur vaut `NULL`, mais ne **devrait pas**. arguments: - name: colname type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant une valeur à vérifier. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant une valeur à vérifier. - name: df3_should_null description: | - Macro de test de la `df2.df3_indicateur` par rapport à `public.indicateur`. + Macro de test de la `df2.df3_indicateur` par rapport à `public.df1_indicateur`. Indique lorsqu'une valeur **n'est pas** `NULL`, mais **devrait l'être**. arguments: - name: colname type: string - description: Nom de la colonne de `df2.df3_indicateur` et `public.indicateur` contenant une valeur à vérifier. + description: Nom de la colonne de `df2.df3_indicateur` et `public.df1_indicateur` contenant une valeur à vérifier. diff --git a/data_management/data_factory/models/analysis/compare_ta_chantier.sql b/data_management/data_factory/models/analysis/compare_ta_chantier.sql new file mode 100644 index 000000000..c75215150 --- /dev/null +++ b/data_management/data_factory/models/analysis/compare_ta_chantier.sql @@ -0,0 +1,41 @@ +with compare as (select + a.id, a.territoire_code, a.taux_avancement as ta, + b.taux_avancement as nouveau_ta +from df1_chantier a +left join df3_chantier b on a.id=b.id and a.territoire_code=b.territoire_code +order by id, territoire_code), +compute_ratio as ( +select *, +case + -- Both TA are null + when coalesce(ta,nouveau_ta) is null then 0 + -- Only old_ta is null + when (ta is null) or (nouveau_ta is null) then null + when ta=0 and nouveau_ta=0 then 0 + when ta=0 and nouveau_ta<>0 then null + when nouveau_ta-ta>=0 then (nouveau_ta-ta)/ta + when nouveau_ta-ta<0 then (ta-nouveau_ta)/ta + else 100000 +end as ratio +from compare), +set_to_buckets as ( + + +select *, +case + when ratio = 0 then '1. exact' + when ratio < 0.01 then '2. approx' + when ratio < 0.2 then '3. <20' + when ratio > 1 then '5. >100' + else '4. 200 then null + when nouveau_ta-ta>=0 then (nouveau_ta-ta)/ta + when nouveau_ta-ta<0 then (ta-nouveau_ta)/ta + else 100000 +end as ratio +from compare), +set_to_buckets as ( + + +select *, +case + when ratio = 0 then '1. exact' + when ratio < 0.01 then '2. approx' + when ratio < 0.2 then '3. <20' + when ratio > 1 then '5. >100' + else '4. 20 TA = NULL - case when vig=vca then null else round((100*(vaca-vig)/(vca-vig))::numeric, {{ var('ta_decimales') }}) end as unbounded_taa, + case + when vig=vca then null + when b.tendance='HAUSSE' and vaca>=vca then 100 + when b.tendance='BAISSE' and vaca<=vca then 100 + else round((100*(vaca-vig)/(vca-vig))::numeric, {{ var('ta_decimales') }}) + end as unbounded_taa, -- TAG: Si la VI=VC-> TA = NULL - case when vig=vcg then null else round((100*(vacg-vig)/(vcg-vig))::numeric, {{ var('ta_decimales') }}) end as unbounded_tag - from {{ ref('merge_computed_values') }} + case + when vig=vcg then null + when b.tendance='HAUSSE' and vacg>=vcg then 100 + when b.tendance='BAISSE' and vacg<=vcg then 100 + else round((100*(vacg-vig)/(vcg-vig))::numeric, {{ var('ta_decimales') }}) + end as unbounded_tag + from {{ ref('merge_computed_values') }} a + left join {{ ref('metadata_parametrage_indicateurs') }} b on a.indic_id=b.indic_id ), -- Compute bounded TA get_bounded_ta as ( diff --git a/data_management/data_factory/models/df3/5_exposition/df3_chantier.sql b/data_management/data_factory/models/df3/5_exposition/df3_chantier.sql index d631439fe..56941aa68 100644 --- a/data_management/data_factory/models/df3/5_exposition/df3_chantier.sql +++ b/data_management/data_factory/models/df3/5_exposition/df3_chantier.sql @@ -66,6 +66,23 @@ select a.chantier_id, bool_or(meteo is not null) filter (where a.maille='NAT') as has_meteo_nat from synthese_triee_par_date a group by chantier_id +), + +-- On récupère les directeurs des directions porteuses de chaque chantier +ch_unnest_porteurs_dac as ( +select chantier_id, unnest(string_to_array("porteur_ids_DAC", ' | ')) as pi from {{ ref('metadata_chantiers') }} mc +), ch_unnest_porteurs_dac_pnames as ( +select a.*, mp.porteur_directeur, mp.porteur_short from ch_unnest_porteurs_dac a +left join {{ ref('metadata_porteurs') }} mp on a.pi=mp.porteur_id +), +ch_unnest_porteurs_dac_pnames_agg as ( +select + chantier_id, + array_agg(pi) p_id, + array_agg(porteur_directeur) p_directeurs, + array_agg(porteur_short) p_shorts + from ch_unnest_porteurs_dac_pnames +group by chantier_id ) @@ -78,9 +95,11 @@ select t.nom as territoire_nom, string_to_array(ch_per, ' | ') as perimetre_ids, z.zone_type as "maille", - string_to_array("porteur_directeur" , ' | ') as directeurs_administration_centrale, + -- coalesce with empty array + coalesce(p_names.p_directeurs, string_to_array('','')) as directeurs_administration_centrale, string_to_array("porteur_ids_noDAC" , ' | ') as ministeres, - string_to_array("porteur_shorts_DAC" , ' | ') as directions_administration_centrale, + -- coalesce with empty array + coalesce(p_names.p_shorts, string_to_array('','')) as directions_administration_centrale, string_to_array("ch_dp" , ' | ') as directeurs_projet, sr.meteo as meteo, ax.axe_name as axe, @@ -101,6 +120,7 @@ from {{ ref('metadata_chantiers') }} mc cross join {{ source('db_schema_public', 'territoire') }} t left join {{ ref('metadata_zones') }} z on z.zone_id=t.zone_id left join {{ ref('metadata_porteurs') }} po on mc."porteur_ids_DAC"=po.porteur_id +left join ch_unnest_porteurs_dac_pnames_agg p_names on mc.chantier_id=p_names.chantier_id left join (select * from synthese_triee_par_date where row_id_by_date_meteo_desc=1) sr ON sr.chantier_id =mc.chantier_id AND diff --git a/data_management/data_factory/models/exposition/chantier.sql b/data_management/data_factory/models/exposition/chantier.sql index 8024b86eb..3d1de286e 100644 --- a/data_management/data_factory/models/exposition/chantier.sql +++ b/data_management/data_factory/models/exposition/chantier.sql @@ -1,147 +1,3 @@ -with +{{config(materialized = 'view')}} -dfakto_chantier as ( - - SELECT fact_progress_chantier.tree_node_id, - fact_progress_chantier.avancement_borne, - fact_progress_chantier.avancement_precedent, - dim_tree_nodes.code_chantier, - dim_tree_nodes.zone_code as zone_id, - dim_structures.nom as structure_nom - FROM {{ ref('stg_dfakto__fact_progress_chantiers') }} fact_progress_chantier - JOIN {{ ref('stg_dfakto__dim_tree_nodes') }} dim_tree_nodes ON fact_progress_chantier.tree_node_id = dim_tree_nodes.id - JOIN {{ ref('stg_dfakto__dim_structures') }} dim_structures ON dim_tree_nodes.structure_id = dim_structures.id - -), - -chantier_est_barometre as ( - SELECT m_indicateurs.chantier_id, - bool_or(m_indicateurs.est_barometre) as est_barometre - FROM {{ ref('stg_ppg_metadata__indicateurs') }} m_indicateurs - GROUP BY m_indicateurs.chantier_id -), - -synthese_triee_par_date as ( - SELECT ROW_NUMBER() OVER (PARTITION BY chantier_id, code_insee, maille ORDER BY date_meteo DESC) AS row_id_by_date_meteo_desc, - chantier_id, - code_insee, - maille, - meteo, - date_meteo - FROM {{ ref('synthese_des_resultats') }} -), - -chantier_taux_avancement_departemental as ( - SELECT d_chantier.code_chantier, - count(*) > 0 as a_taux_avancement_departemental - FROM dfakto_chantier d_chantier - LEFT JOIN {{ ref('stg_ppg_metadata__zones') }} ppg_zones ON d_chantier.zone_id = ppg_zones.id - WHERE avancement_borne is not NULL - AND ppg_zones.maille = 'DEPT' - GROUP BY d_chantier.code_chantier -), - -chantier_taux_avancement_regional as ( - SELECT d_chantier.code_chantier, - count(*) > 0 as a_taux_avancement_regional - FROM dfakto_chantier d_chantier - LEFT JOIN {{ ref('stg_ppg_metadata__zones') }} ppg_zones ON d_chantier.zone_id = ppg_zones.id - WHERE avancement_borne is not NULL - AND ppg_zones.maille = 'REG' - GROUP BY d_chantier.code_chantier -), - -chantier_meteo_departemental as ( - SELECT chantier_id, - count(*) > 0 as a_meteo_departemental - FROM synthese_triee_par_date - WHERE meteo is not NULL - AND maille = 'DEPT' - GROUP BY chantier_id -), - -chantier_meteo_regional as ( - SELECT chantier_id, - count(*) > 0 as a_meteo_regional - FROM synthese_triee_par_date - WHERE meteo is not NULL - AND maille = 'REG' - GROUP BY chantier_id -), - -indicateurs_zones AS ( - SELECT DISTINCT - spmi.id as indic_id, - chantier_id, - zones.id as zone_id - FROM {{ ref('stg_ppg_metadata__indicateurs') }} spmi - CROSS JOIN ( - SELECT DISTINCT - id - FROM {{ ref('stg_ppg_metadata__zones') }} - ) as zones -), -indicateurs_zones_applicables AS ( - SELECT DISTINCT - iz.indic_id, - iz.chantier_id, - iz.zone_id, - COALESCE(iiza.est_applicable, TRUE) AS est_applicable - FROM indicateurs_zones iz - LEFT JOIN {{ ref('int_indicateurs_zones_applicables') }} iiza - ON iz.indic_id = iiza.indic_id AND iz.zone_id = iiza.zone_id -), -chantiers_zones_applicables AS ( - SELECT - chantier_id, - zone_id, - bool_or(est_applicable) as est_applicable - FROM indicateurs_zones_applicables - GROUP BY chantier_id, zone_id -) - -SELECT m_chantiers.id, - m_chantiers.nom, - m_chantiers.code_insee, - d_chantiers.avancement_borne AS taux_avancement, - d_chantiers.avancement_precedent AS taux_avancement_precedent, - m_chantiers.nom AS territoire_nom, - m_chantiers.perimetre_ids, - m_chantiers.maille, - m_chantiers.directeurs_administration_centrale, - m_chantiers.ministeres, - m_chantiers.directions_administration_centrale, - m_chantiers.directeurs_projet, - synthese_triee_par_date.meteo AS meteo, - m_axes.nom AS axe, - m_ppgs.nom AS ppg, - m_chantiers.directeurs_projet_mails, - chantier_est_barometre.est_barometre, - m_chantiers.est_territorialise, - CONCAT(m_chantiers.maille, '-', m_chantiers.code_insee) as territoire_code, - LOWER(m_chantiers.ate)::type_ate as ate, - chantier_ta_dep.a_taux_avancement_departemental, - chantier_ta_reg.a_taux_avancement_regional, - chantier_meteo_dep.a_meteo_departemental, - chantier_meteo_reg.a_meteo_regional, - chantier_za.est_applicable, - false AS a_supprimer - FROM {{ ref('int_chantiers_with_mailles_and_territoires') }} m_chantiers - LEFT JOIN dfakto_chantier d_chantiers - ON m_chantiers.id = d_chantiers.code_chantier - AND m_chantiers.zone_id = d_chantiers.zone_id - AND d_chantiers.structure_nom IN ('Région', 'Département', 'Chantier') - LEFT JOIN {{ ref('stg_ppg_metadata__ppgs') }} m_ppgs ON m_ppgs.id = m_chantiers.ppg_id - LEFT JOIN {{ ref('stg_ppg_metadata__axes') }} m_axes ON m_axes.id = m_ppgs.axe_id - LEFT JOIN chantier_est_barometre on m_chantiers.id = chantier_est_barometre.chantier_id - LEFT JOIN synthese_triee_par_date - ON synthese_triee_par_date.chantier_id = m_chantiers.id - AND synthese_triee_par_date.maille = m_chantiers.maille - AND synthese_triee_par_date.code_insee = m_chantiers.code_insee - LEFT JOIN chantier_taux_avancement_departemental chantier_ta_dep ON m_chantiers.id = chantier_ta_dep.code_chantier - LEFT JOIN chantier_taux_avancement_regional chantier_ta_reg ON m_chantiers.id = chantier_ta_reg.code_chantier - LEFT JOIN chantier_meteo_departemental chantier_meteo_dep ON m_chantiers.id = chantier_meteo_dep.chantier_id - LEFT JOIN chantier_meteo_regional chantier_meteo_reg ON m_chantiers.id = chantier_meteo_reg.chantier_id - LEFT JOIN chantiers_zones_applicables chantier_za ON chantier_za.chantier_id = m_chantiers.id AND chantier_za.zone_id = m_chantiers.zone_id - WHERE synthese_triee_par_date.row_id_by_date_meteo_desc = 1 - OR synthese_triee_par_date.row_id_by_date_meteo_desc IS NULL +SELECT * from {{ ref('df1_chantier') }} \ No newline at end of file diff --git a/data_management/data_factory/models/exposition/df1_chantier.sql b/data_management/data_factory/models/exposition/df1_chantier.sql new file mode 100644 index 000000000..8024b86eb --- /dev/null +++ b/data_management/data_factory/models/exposition/df1_chantier.sql @@ -0,0 +1,147 @@ +with + +dfakto_chantier as ( + + SELECT fact_progress_chantier.tree_node_id, + fact_progress_chantier.avancement_borne, + fact_progress_chantier.avancement_precedent, + dim_tree_nodes.code_chantier, + dim_tree_nodes.zone_code as zone_id, + dim_structures.nom as structure_nom + FROM {{ ref('stg_dfakto__fact_progress_chantiers') }} fact_progress_chantier + JOIN {{ ref('stg_dfakto__dim_tree_nodes') }} dim_tree_nodes ON fact_progress_chantier.tree_node_id = dim_tree_nodes.id + JOIN {{ ref('stg_dfakto__dim_structures') }} dim_structures ON dim_tree_nodes.structure_id = dim_structures.id + +), + +chantier_est_barometre as ( + SELECT m_indicateurs.chantier_id, + bool_or(m_indicateurs.est_barometre) as est_barometre + FROM {{ ref('stg_ppg_metadata__indicateurs') }} m_indicateurs + GROUP BY m_indicateurs.chantier_id +), + +synthese_triee_par_date as ( + SELECT ROW_NUMBER() OVER (PARTITION BY chantier_id, code_insee, maille ORDER BY date_meteo DESC) AS row_id_by_date_meteo_desc, + chantier_id, + code_insee, + maille, + meteo, + date_meteo + FROM {{ ref('synthese_des_resultats') }} +), + +chantier_taux_avancement_departemental as ( + SELECT d_chantier.code_chantier, + count(*) > 0 as a_taux_avancement_departemental + FROM dfakto_chantier d_chantier + LEFT JOIN {{ ref('stg_ppg_metadata__zones') }} ppg_zones ON d_chantier.zone_id = ppg_zones.id + WHERE avancement_borne is not NULL + AND ppg_zones.maille = 'DEPT' + GROUP BY d_chantier.code_chantier +), + +chantier_taux_avancement_regional as ( + SELECT d_chantier.code_chantier, + count(*) > 0 as a_taux_avancement_regional + FROM dfakto_chantier d_chantier + LEFT JOIN {{ ref('stg_ppg_metadata__zones') }} ppg_zones ON d_chantier.zone_id = ppg_zones.id + WHERE avancement_borne is not NULL + AND ppg_zones.maille = 'REG' + GROUP BY d_chantier.code_chantier +), + +chantier_meteo_departemental as ( + SELECT chantier_id, + count(*) > 0 as a_meteo_departemental + FROM synthese_triee_par_date + WHERE meteo is not NULL + AND maille = 'DEPT' + GROUP BY chantier_id +), + +chantier_meteo_regional as ( + SELECT chantier_id, + count(*) > 0 as a_meteo_regional + FROM synthese_triee_par_date + WHERE meteo is not NULL + AND maille = 'REG' + GROUP BY chantier_id +), + +indicateurs_zones AS ( + SELECT DISTINCT + spmi.id as indic_id, + chantier_id, + zones.id as zone_id + FROM {{ ref('stg_ppg_metadata__indicateurs') }} spmi + CROSS JOIN ( + SELECT DISTINCT + id + FROM {{ ref('stg_ppg_metadata__zones') }} + ) as zones +), +indicateurs_zones_applicables AS ( + SELECT DISTINCT + iz.indic_id, + iz.chantier_id, + iz.zone_id, + COALESCE(iiza.est_applicable, TRUE) AS est_applicable + FROM indicateurs_zones iz + LEFT JOIN {{ ref('int_indicateurs_zones_applicables') }} iiza + ON iz.indic_id = iiza.indic_id AND iz.zone_id = iiza.zone_id +), +chantiers_zones_applicables AS ( + SELECT + chantier_id, + zone_id, + bool_or(est_applicable) as est_applicable + FROM indicateurs_zones_applicables + GROUP BY chantier_id, zone_id +) + +SELECT m_chantiers.id, + m_chantiers.nom, + m_chantiers.code_insee, + d_chantiers.avancement_borne AS taux_avancement, + d_chantiers.avancement_precedent AS taux_avancement_precedent, + m_chantiers.nom AS territoire_nom, + m_chantiers.perimetre_ids, + m_chantiers.maille, + m_chantiers.directeurs_administration_centrale, + m_chantiers.ministeres, + m_chantiers.directions_administration_centrale, + m_chantiers.directeurs_projet, + synthese_triee_par_date.meteo AS meteo, + m_axes.nom AS axe, + m_ppgs.nom AS ppg, + m_chantiers.directeurs_projet_mails, + chantier_est_barometre.est_barometre, + m_chantiers.est_territorialise, + CONCAT(m_chantiers.maille, '-', m_chantiers.code_insee) as territoire_code, + LOWER(m_chantiers.ate)::type_ate as ate, + chantier_ta_dep.a_taux_avancement_departemental, + chantier_ta_reg.a_taux_avancement_regional, + chantier_meteo_dep.a_meteo_departemental, + chantier_meteo_reg.a_meteo_regional, + chantier_za.est_applicable, + false AS a_supprimer + FROM {{ ref('int_chantiers_with_mailles_and_territoires') }} m_chantiers + LEFT JOIN dfakto_chantier d_chantiers + ON m_chantiers.id = d_chantiers.code_chantier + AND m_chantiers.zone_id = d_chantiers.zone_id + AND d_chantiers.structure_nom IN ('Région', 'Département', 'Chantier') + LEFT JOIN {{ ref('stg_ppg_metadata__ppgs') }} m_ppgs ON m_ppgs.id = m_chantiers.ppg_id + LEFT JOIN {{ ref('stg_ppg_metadata__axes') }} m_axes ON m_axes.id = m_ppgs.axe_id + LEFT JOIN chantier_est_barometre on m_chantiers.id = chantier_est_barometre.chantier_id + LEFT JOIN synthese_triee_par_date + ON synthese_triee_par_date.chantier_id = m_chantiers.id + AND synthese_triee_par_date.maille = m_chantiers.maille + AND synthese_triee_par_date.code_insee = m_chantiers.code_insee + LEFT JOIN chantier_taux_avancement_departemental chantier_ta_dep ON m_chantiers.id = chantier_ta_dep.code_chantier + LEFT JOIN chantier_taux_avancement_regional chantier_ta_reg ON m_chantiers.id = chantier_ta_reg.code_chantier + LEFT JOIN chantier_meteo_departemental chantier_meteo_dep ON m_chantiers.id = chantier_meteo_dep.chantier_id + LEFT JOIN chantier_meteo_regional chantier_meteo_reg ON m_chantiers.id = chantier_meteo_reg.chantier_id + LEFT JOIN chantiers_zones_applicables chantier_za ON chantier_za.chantier_id = m_chantiers.id AND chantier_za.zone_id = m_chantiers.zone_id + WHERE synthese_triee_par_date.row_id_by_date_meteo_desc = 1 + OR synthese_triee_par_date.row_id_by_date_meteo_desc IS NULL diff --git a/data_management/data_factory/models/exposition/df1_indicateur.sql b/data_management/data_factory/models/exposition/df1_indicateur.sql new file mode 100644 index 000000000..6a87eecc9 --- /dev/null +++ b/data_management/data_factory/models/exposition/df1_indicateur.sql @@ -0,0 +1,51 @@ +SELECT m_indicateurs.id, + m_indicateurs.nom, + m_indicateurs.chantier_id, + d_indicateurs.objectif_valeur_cible as objectif_valeur_cible, + d_indicateurs.objectif_taux_avancement as objectif_taux_avancement, + m_indicateurs.indicateur_type_id as type_id, + indicateur_types.nom AS type_nom, + m_indicateurs.est_barometre, + m_indicateurs.est_phare, + d_indicateurs.date_valeur_actuelle as date_valeur_actuelle, + d_indicateurs.date_valeur_initiale as date_valeur_initiale, + d_indicateurs.valeur_actuelle as valeur_actuelle, + d_indicateurs.valeur_initiale as valeur_initiale, + chantiers_ayant_des_indicateurs.code_insee, + chantiers_ayant_des_indicateurs.maille, + chantiers_ayant_des_indicateurs.territoire_nom, + d_indicateurs.evolution_valeur_actuelle as evolution_valeur_actuelle, + d_indicateurs.evolution_date_valeur_actuelle as evolution_date_valeur_actuelle, + m_indicateurs.description, + m_indicateurs.source, + m_indicateurs.mode_de_calcul, + m_indicateurs.unite as unite_mesure, + CONCAT(chantiers_ayant_des_indicateurs.maille, '-', chantiers_ayant_des_indicateurs.code_insee) as territoire_code, + parametrage_indicateurs.poids_pourcent_dept as ponderation_dept, + parametrage_indicateurs.poids_pourcent_nat as ponderation_nat, + parametrage_indicateurs.poids_pourcent_reg as ponderation_reg, + d_indicateurs.objectif_date_valeur_cible as objectif_date_valeur_cible, + d_indicateurs.objectif_valeur_cible_intermediaire, + d_indicateurs.objectif_taux_avancement_intermediaire, + d_indicateurs.objectif_date_valeur_cible_intermediaire, + COALESCE(indicateur_zone.est_applicable, true) AS est_applicable, + last_update_indic_zone.dernier_import_date, + last_update_indic_zone.dernier_import_rapport_id, + last_update_indic_zone.dernier_import_auteur, + last_update_indic.dernier_import_date_indic, + last_update_indic.dernier_import_rapport_id_indic, + last_update_indic.dernier_import_auteur_indic, + false AS a_supprimer +FROM {{ ref('stg_ppg_metadata__indicateurs') }} m_indicateurs + JOIN {{ ref('int_chantiers_with_mailles_and_territoires') }} chantiers_ayant_des_indicateurs ON m_indicateurs.chantier_id = chantiers_ayant_des_indicateurs.id + LEFT JOIN {{ ref('stg_ppg_metadata__indicateur_types') }} indicateur_types ON indicateur_types.id = m_indicateurs.indicateur_type_id + LEFT JOIN {{ ref('int_dfakto_indicateurs_metrics') }} d_indicateurs + ON m_indicateurs.id = d_indicateurs.effect_id + AND chantiers_ayant_des_indicateurs.id = d_indicateurs.code_chantier -- TODO: a supprimer car temporaire pour bug dfakto avec ligne 43 du fichier int_dfakto_indicateurs_metrics.sql + AND chantiers_ayant_des_indicateurs.zone_id = d_indicateurs.zone_code + AND d_indicateurs.nom_structure IN ('Département', 'Région', 'Chantier') + LEFT JOIN {{ ref('stg_ppg_metadata__parametrage_indicateurs') }} parametrage_indicateurs ON parametrage_indicateurs.indicateur_id = m_indicateurs.id + LEFT JOIN {{ ref('int_indicateurs_zones_applicables')}} indicateur_zone ON indicateur_zone.indic_id = m_indicateurs.id AND indicateur_zone.zone_id = chantiers_ayant_des_indicateurs.zone_id + LEFT JOIN {{ ref('last_update_indic_zone') }} last_update_indic_zone ON m_indicateurs.id=last_update_indic_zone.indic_id AND CONCAT(chantiers_ayant_des_indicateurs.maille, '-', chantiers_ayant_des_indicateurs.code_insee) =last_update_indic_zone.territoire_code + LEFT JOIN {{ ref('last_update_indic') }} last_update_indic ON m_indicateurs.id=last_update_indic.indic_id +ORDER BY m_indicateurs.nom, chantiers_ayant_des_indicateurs.maille, chantiers_ayant_des_indicateurs.code_insee, d_indicateurs.date_valeur_actuelle DESC diff --git a/data_management/data_factory/models/exposition/indicateur.sql b/data_management/data_factory/models/exposition/indicateur.sql index 6a87eecc9..bbc4a51a8 100644 --- a/data_management/data_factory/models/exposition/indicateur.sql +++ b/data_management/data_factory/models/exposition/indicateur.sql @@ -1,51 +1,3 @@ -SELECT m_indicateurs.id, - m_indicateurs.nom, - m_indicateurs.chantier_id, - d_indicateurs.objectif_valeur_cible as objectif_valeur_cible, - d_indicateurs.objectif_taux_avancement as objectif_taux_avancement, - m_indicateurs.indicateur_type_id as type_id, - indicateur_types.nom AS type_nom, - m_indicateurs.est_barometre, - m_indicateurs.est_phare, - d_indicateurs.date_valeur_actuelle as date_valeur_actuelle, - d_indicateurs.date_valeur_initiale as date_valeur_initiale, - d_indicateurs.valeur_actuelle as valeur_actuelle, - d_indicateurs.valeur_initiale as valeur_initiale, - chantiers_ayant_des_indicateurs.code_insee, - chantiers_ayant_des_indicateurs.maille, - chantiers_ayant_des_indicateurs.territoire_nom, - d_indicateurs.evolution_valeur_actuelle as evolution_valeur_actuelle, - d_indicateurs.evolution_date_valeur_actuelle as evolution_date_valeur_actuelle, - m_indicateurs.description, - m_indicateurs.source, - m_indicateurs.mode_de_calcul, - m_indicateurs.unite as unite_mesure, - CONCAT(chantiers_ayant_des_indicateurs.maille, '-', chantiers_ayant_des_indicateurs.code_insee) as territoire_code, - parametrage_indicateurs.poids_pourcent_dept as ponderation_dept, - parametrage_indicateurs.poids_pourcent_nat as ponderation_nat, - parametrage_indicateurs.poids_pourcent_reg as ponderation_reg, - d_indicateurs.objectif_date_valeur_cible as objectif_date_valeur_cible, - d_indicateurs.objectif_valeur_cible_intermediaire, - d_indicateurs.objectif_taux_avancement_intermediaire, - d_indicateurs.objectif_date_valeur_cible_intermediaire, - COALESCE(indicateur_zone.est_applicable, true) AS est_applicable, - last_update_indic_zone.dernier_import_date, - last_update_indic_zone.dernier_import_rapport_id, - last_update_indic_zone.dernier_import_auteur, - last_update_indic.dernier_import_date_indic, - last_update_indic.dernier_import_rapport_id_indic, - last_update_indic.dernier_import_auteur_indic, - false AS a_supprimer -FROM {{ ref('stg_ppg_metadata__indicateurs') }} m_indicateurs - JOIN {{ ref('int_chantiers_with_mailles_and_territoires') }} chantiers_ayant_des_indicateurs ON m_indicateurs.chantier_id = chantiers_ayant_des_indicateurs.id - LEFT JOIN {{ ref('stg_ppg_metadata__indicateur_types') }} indicateur_types ON indicateur_types.id = m_indicateurs.indicateur_type_id - LEFT JOIN {{ ref('int_dfakto_indicateurs_metrics') }} d_indicateurs - ON m_indicateurs.id = d_indicateurs.effect_id - AND chantiers_ayant_des_indicateurs.id = d_indicateurs.code_chantier -- TODO: a supprimer car temporaire pour bug dfakto avec ligne 43 du fichier int_dfakto_indicateurs_metrics.sql - AND chantiers_ayant_des_indicateurs.zone_id = d_indicateurs.zone_code - AND d_indicateurs.nom_structure IN ('Département', 'Région', 'Chantier') - LEFT JOIN {{ ref('stg_ppg_metadata__parametrage_indicateurs') }} parametrage_indicateurs ON parametrage_indicateurs.indicateur_id = m_indicateurs.id - LEFT JOIN {{ ref('int_indicateurs_zones_applicables')}} indicateur_zone ON indicateur_zone.indic_id = m_indicateurs.id AND indicateur_zone.zone_id = chantiers_ayant_des_indicateurs.zone_id - LEFT JOIN {{ ref('last_update_indic_zone') }} last_update_indic_zone ON m_indicateurs.id=last_update_indic_zone.indic_id AND CONCAT(chantiers_ayant_des_indicateurs.maille, '-', chantiers_ayant_des_indicateurs.code_insee) =last_update_indic_zone.territoire_code - LEFT JOIN {{ ref('last_update_indic') }} last_update_indic ON m_indicateurs.id=last_update_indic.indic_id -ORDER BY m_indicateurs.nom, chantiers_ayant_des_indicateurs.maille, chantiers_ayant_des_indicateurs.code_insee, d_indicateurs.date_valeur_actuelle DESC +{{config(materialized = 'view')}} + +SELECT * from {{ ref('df1_indicateur') }} \ No newline at end of file diff --git a/data_management/data_factory/models/exposition/schema.yml b/data_management/data_factory/models/exposition/schema.yml index 594e742df..5a47f8708 100644 --- a/data_management/data_factory/models/exposition/schema.yml +++ b/data_management/data_factory/models/exposition/schema.yml @@ -2,6 +2,7 @@ version: 2 models: - name: chantier + description: "Une vue sur `df1_chantier` ou `df3_chantier`" columns: - name: id description: "source de données à date du 06/07/2023 : ppg_metadata.view_meta_chantier" @@ -84,6 +85,7 @@ models: - name: date description: "source de données à date du 07/07/2023 : dfakto.ps_view_data_financials" - name: indicateur + description: "Une vue sur `df1_indicateur` ou `df3_indicateur`" columns: - name: id description: "source de données à date du 07/07/2023 : ppg_metadata.view_meta_indicateur" @@ -293,3 +295,7 @@ models: description: "source de données à date du 07/07/2023 : dfakto.ps_view_data_financials" - name: auteur description: "source de données à date du 07/07/2023 : toujours null" + - name: df1_indicateur + description: "Table des indicateurs générée par la datafactory v1" + - name: df1_chantier + description: "Table des chantiers générée par la datafactory v1" \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_different.sql b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_different.sql index f5cba7547..15e624004 100644 --- a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_different.sql +++ b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_asuppr"]) }} {{ df3_not_equal_bool('a_supprimer') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_not_null.sql b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_not_null.sql index e95ef959f..b779cfe4b 100644 --- a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_not_null.sql +++ b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_asuppr"]) }} {{ df3_should_notnull('a_supprimer') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_null.sql b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_null.sql index 2f862f071..7a9eb9306 100644 --- a/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_null.sql +++ b/data_management/data_factory/tests/df3/a_suppr/df3_asuppr_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_asuppr"]) }} {{ df3_should_null('a_supprimer') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/baro/df3_baro_different.sql b/data_management/data_factory/tests/df3/baro/df3_baro_different.sql index 7dd60979a..c26c6a4ad 100644 --- a/data_management/data_factory/tests/df3/baro/df3_baro_different.sql +++ b/data_management/data_factory/tests/df3/baro/df3_baro_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_baro"]) }} {{ df3_not_equal_bool('est_barometre') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/baro/df3_baro_should_not_null.sql b/data_management/data_factory/tests/df3/baro/df3_baro_should_not_null.sql index 916fbe387..58d6d9935 100644 --- a/data_management/data_factory/tests/df3/baro/df3_baro_should_not_null.sql +++ b/data_management/data_factory/tests/df3/baro/df3_baro_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_baro"]) }} {{ df3_should_notnull('est_barometre') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/baro/df3_baro_should_null.sql b/data_management/data_factory/tests/df3/baro/df3_baro_should_null.sql index 992639d41..3faf1f3fb 100644 --- a/data_management/data_factory/tests/df3/baro/df3_baro_should_null.sql +++ b/data_management/data_factory/tests/df3/baro/df3_baro_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_baro"]) }} {{ df3_should_null('est_barometre') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/ta/df3_ta_different.sql b/data_management/data_factory/tests/df3/ta/df3_ta_different.sql index 38c2ca76b..f7f4b2149 100644 --- a/data_management/data_factory/tests/df3/ta/df3_ta_different.sql +++ b/data_management/data_factory/tests/df3/ta/df3_ta_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_ta"]) }} {{ df3_not_equal('objectif_taux_avancement') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/ta/df3_ta_should_not_null.sql b/data_management/data_factory/tests/df3/ta/df3_ta_should_not_null.sql index 8ef01929e..7568f0ed7 100644 --- a/data_management/data_factory/tests/df3/ta/df3_ta_should_not_null.sql +++ b/data_management/data_factory/tests/df3/ta/df3_ta_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_ta"]) }} {{ df3_should_notnull('objectif_taux_avancement') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/ta/df3_ta_should_null.sql b/data_management/data_factory/tests/df3/ta/df3_ta_should_null.sql index 9e4dc85d6..6e508db85 100644 --- a/data_management/data_factory/tests/df3/ta/df3_ta_should_null.sql +++ b/data_management/data_factory/tests/df3/ta/df3_ta_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_ta"]) }} {{ df3_should_null('objectif_taux_avancement') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_date_late.sql b/data_management/data_factory/tests/df3/va/df3_va_date_late.sql index 5df816c98..fa4a87b2e 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_date_late.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_date_late.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_date_late('date_valeur_actuelle', 'TRUE') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_date_should_not_null.sql b/data_management/data_factory/tests/df3/va/df3_va_date_should_not_null.sql index d2b5107fc..486389a40 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_date_should_not_null.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_date_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_should_notnull('date_valeur_actuelle') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_date_should_null.sql b/data_management/data_factory/tests/df3/va/df3_va_date_should_null.sql index 7f0b98891..4f7caec61 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_date_should_null.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_date_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_should_null('date_valeur_actuelle') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_date_soon.sql b/data_management/data_factory/tests/df3/va/df3_va_date_soon.sql index 257d25290..71f8c30a3 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_date_soon.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_date_soon.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_date_early('date_valeur_actuelle', 'TRUE') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_different.sql b/data_management/data_factory/tests/df3/va/df3_va_different.sql index 29e340f1e..e348683b3 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_different.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_not_equal('valeur_actuelle') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_should_not_null.sql b/data_management/data_factory/tests/df3/va/df3_va_should_not_null.sql index 04cc4fe07..1fc1e1344 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_should_not_null.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_should_notnull('valeur_actuelle') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va/df3_va_should_null.sql b/data_management/data_factory/tests/df3/va/df3_va_should_null.sql index 57833bed4..fe155aae1 100644 --- a/data_management/data_factory/tests/df3/va/df3_va_should_null.sql +++ b/data_management/data_factory/tests/df3/va/df3_va_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_va"]) }} {{ df3_should_null('valeur_actuelle') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_different.sql b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_different.sql index ee6dac857..91329ae2d 100644 --- a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_different.sql +++ b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vaunnest"]) }} -- TOLERANCE de 1 pourcent (cf fin du test) diff --git a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_missing.sql b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_missing.sql index 5d6b976d7..c8ebeeaf4 100644 --- a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_missing.sql +++ b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_missing.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vaunnest"]) }} -- Ce test va vérifier si UNE VA est assignée à CHAQUE date diff --git a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_suppl.sql b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_suppl.sql index 4b89316b5..85a711e49 100644 --- a/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_suppl.sql +++ b/data_management/data_factory/tests/df3/va_unnest/df3_vaunnest_suppl.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vaunnest"]) }} -- Ce test va vérifier si des VA sont assignées à des dates diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_date_late.sql b/data_management/data_factory/tests/df3/vca/df3_vca_date_late.sql index ec88644c4..600cd25c6 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_date_late.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_date_late.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_date_late('objectif_date_valeur_cible_intermediaire', 'TRUE') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_date_should_not_null.sql b/data_management/data_factory/tests/df3/vca/df3_vca_date_should_not_null.sql index 2ef85a0b1..866472164 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_date_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_date_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_should_notnull('objectif_date_valeur_cible_intermediaire') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_date_should_null.sql b/data_management/data_factory/tests/df3/vca/df3_vca_date_should_null.sql index a0a2d858d..ef6be50a4 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_date_should_null.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_date_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_should_null('objectif_date_valeur_cible_intermediaire') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_date_soon.sql b/data_management/data_factory/tests/df3/vca/df3_vca_date_soon.sql index 5015f730b..31872a660 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_date_soon.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_date_soon.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_date_early('objectif_date_valeur_cible_intermediaire', 'TRUE') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_different.sql b/data_management/data_factory/tests/df3/vca/df3_vca_different.sql index 20e02fb73..92daef5e3 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_different.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_not_equal('objectif_valeur_cible_intermediaire') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_should_not_null.sql b/data_management/data_factory/tests/df3/vca/df3_vca_should_not_null.sql index d05dfa97e..2cdaca980 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_should_notnull('objectif_valeur_cible_intermediaire') }} diff --git a/data_management/data_factory/tests/df3/vca/df3_vca_should_null.sql b/data_management/data_factory/tests/df3/vca/df3_vca_should_null.sql index 1edc8b79c..624580500 100644 --- a/data_management/data_factory/tests/df3/vca/df3_vca_should_null.sql +++ b/data_management/data_factory/tests/df3/vca/df3_vca_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vca"]) }} {{ df3_should_null('objectif_valeur_cible_intermediaire') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_late.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_late.sql index 9b3410ddd..08db4c904 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_late.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_late.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_date_late('objectif_date_valeur_cible', 'TRUE') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_not_null.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_not_null.sql index e3d1776ea..8347cd4bd 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_should_notnull('objectif_date_valeur_cible') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_null.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_null.sql index 359d387ef..21bfbd6de 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_null.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_should_null('objectif_date_valeur_cible') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_soon.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_soon.sql index 3d61a84ce..7781c0b0c 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_date_soon.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_date_soon.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_date_early('objectif_date_valeur_cible', 'TRUE') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_different.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_different.sql index 5a69fcaa5..0b14a2591 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_different.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_not_equal('objectif_valeur_cible') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_should_not_null.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_should_not_null.sql index d246ee775..5a64d9fdc 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_should_notnull('objectif_valeur_cible') }} diff --git a/data_management/data_factory/tests/df3/vcg/df3_vcg_should_null.sql b/data_management/data_factory/tests/df3/vcg/df3_vcg_should_null.sql index d522e819e..58aeac140 100644 --- a/data_management/data_factory/tests/df3/vcg/df3_vcg_should_null.sql +++ b/data_management/data_factory/tests/df3/vcg/df3_vcg_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vcg"]) }} {{ df3_should_null('objectif_valeur_cible') }} diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_date_late.sql b/data_management/data_factory/tests/df3/vi/df3_vi_date_late.sql index e6e542659..704a82e22 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_date_late.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_date_late.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_date_late('date_valeur_initiale', 'TRUE') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_date_should_not_null.sql b/data_management/data_factory/tests/df3/vi/df3_vi_date_should_not_null.sql index ddb20a855..a70dc44f5 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_date_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_date_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_should_notnull('date_valeur_initiale') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_date_should_null.sql b/data_management/data_factory/tests/df3/vi/df3_vi_date_should_null.sql index f2ee80cbd..b817d387b 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_date_should_null.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_date_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_should_null('date_valeur_initiale') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_date_soon.sql b/data_management/data_factory/tests/df3/vi/df3_vi_date_soon.sql index 94b28c9d4..f28713247 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_date_soon.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_date_soon.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_date_early('date_valeur_initiale', 'TRUE') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_different.sql b/data_management/data_factory/tests/df3/vi/df3_vi_different.sql index ca72967fe..1dea6fa16 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_different.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_not_equal('valeur_initiale') }} diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_should_not_null.sql b/data_management/data_factory/tests/df3/vi/df3_vi_should_not_null.sql index a99a3e05f..c0c82753d 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_should_not_null.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_should_notnull('valeur_initiale') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/vi/df3_vi_should_null.sql b/data_management/data_factory/tests/df3/vi/df3_vi_should_null.sql index ae3bb69f8..5098bcdff 100644 --- a/data_management/data_factory/tests/df3/vi/df3_vi_should_null.sql +++ b/data_management/data_factory/tests/df3/vi/df3_vi_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_vi"]) }} {{ df3_should_null('valeur_initiale') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/z_appl/df3_zappl_different.sql b/data_management/data_factory/tests/df3/z_appl/df3_zappl_different.sql index fc61b16f1..012211b40 100644 --- a/data_management/data_factory/tests/df3/z_appl/df3_zappl_different.sql +++ b/data_management/data_factory/tests/df3/z_appl/df3_zappl_different.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_zappl"]) }} {{ df3_not_equal_bool('est_applicable') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_not_null.sql b/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_not_null.sql index 2c98f28fc..d67c0ea9f 100644 --- a/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_not_null.sql +++ b/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_not_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_zappl"]) }} {{ df3_should_notnull('est_applicable') }} \ No newline at end of file diff --git a/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_null.sql b/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_null.sql index ead6b1321..6819da80f 100644 --- a/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_null.sql +++ b/data_management/data_factory/tests/df3/z_appl/df3_zappl_should_null.sql @@ -1,3 +1,4 @@ +-- depends_on: {{ ref('df1_indicateur') }} {{ config(enabled=true, tags=["scope_indicateur", "metric_zappl"]) }} {{ df3_should_null('est_applicable') }} \ No newline at end of file diff --git a/data_management/scripts/migration/README.md b/data_management/scripts/migration/README.md index bd93fc213..43061ae87 100644 --- a/data_management/scripts/migration/README.md +++ b/data_management/scripts/migration/README.md @@ -5,5 +5,5 @@ Pour mettre à jour pilote en local avec la prod: - script 0 et 1 et 2: `/bin/bash scripts/0_prisma_migrate.sh && /bin/bash scripts/1_dump_dfakto.sh && /bin/bash scripts/2_fill_tables_ppg_metadata.sh` - exporter la prod: `/bin/bash dump_prod.sh` - importer en local: `/bin/bash load_to_local.sh` -- ajout utilisateur ditp_admin `INSERT INTO public.utilisateur (id,email,nom,prenom,profil_code,auteur_modification,date_modification,fonction) VALUES ('46eb7569-0644-4ada-be98-abc1e417af57','ditp.admin@example.com','ditp','admin','DITP_ADMIN','Import CSV','2023-06-16 14:21:33.142',NULL);` +- Se connecter avec un utilisateur de prod au profil `DITP_ADMIN`, et le mdp developpeur en local - script 5 et 7: `/bin/bash scripts/5_fill_tables_staging.sh && /bin/bash scripts/7_fill_tables_public.sh` diff --git a/data_management/scripts/migration/dump_prod.sh b/data_management/scripts/migration/dump_prod.sh index fd6efde86..295ebe0f2 100644 --- a/data_management/scripts/migration/dump_prod.sh +++ b/data_management/scripts/migration/dump_prod.sh @@ -8,18 +8,31 @@ export $(cat .env | xargs) export PGPASSWORD="$PWD_PROD" +# Export table with CREATE command pg_dump -h $prod_host -p $prod_port -U $prod_user -d $prod_db \ + -O -v \ -t "public.utilisateur" \ + > out/export_public_utilisateur.sql +# Export table without CREATE command +pg_dump -h $prod_host -p $prod_port -U $prod_user -d $prod_db \ + -O -v \ -t "public.commentaire" \ -t "public.synthese_des_resultats" \ -t "public.rapport_import_mesure_indicateur" \ -t "public.habilitation" \ -a > out/export_public.sql +# Export table with CREATE command pg_dump -h $prod_host -p $prod_port -U $prod_user -d $prod_db \ + -O -v \ -t "raw_data.commentaires" \ + > out/export_raw_commentaire.sql + +# Export table without CREATE command +pg_dump -h $prod_host -p $prod_port -U $prod_user -d $prod_db \ + -O -v \ -t "raw_data.mesure_indicateur" \ - -a > out/export_raw.sql + -a > out/export_raw_mesure_indicateur.sql unset PGPASSWORD diff --git a/data_management/scripts/migration/load_to_local.sh b/data_management/scripts/migration/load_to_local.sh index ee232d40a..447c0bb0f 100644 --- a/data_management/scripts/migration/load_to_local.sh +++ b/data_management/scripts/migration/load_to_local.sh @@ -3,9 +3,11 @@ export $(cat .env | xargs) export PGPASSWORD="$PWD_LOCAL" # Importer les données dans la deuxième base de données echo ">> Load public data" +psql -h $local_host -p $local_port -U $local_user -d $local_db -f out/export_public_utilisateur.sql psql -h $local_host -p $local_port -U $local_user -d $local_db -f out/export_public.sql echo ">> Load raw data" -psql -h $local_host -p $local_port -U $local_user -d $local_db -f out/export_raw.sql +psql -h $local_host -p $local_port -U $local_user -d $local_db -f out/export_raw_commentaire.sql +psql -h $local_host -p $local_port -U $local_user -d $local_db -f out/export_raw_mesure_indicateur.sql unset PGPASSWORD # script 5 7 \ No newline at end of file diff --git a/data_management/scripts/migration/out/.gitignore b/data_management/scripts/migration/out/.gitignore index a37a363bc..798f54c69 100644 --- a/data_management/scripts/migration/out/.gitignore +++ b/data_management/scripts/migration/out/.gitignore @@ -1,2 +1,5 @@ export_public.sql -export_raw.sql \ No newline at end of file +export_raw.sql +export_public_utilisateur.sql +export_raw_commentaire.sql +export_raw_mesure_indicateur.sql \ No newline at end of file