From 7fae32d37661686dc8dee31865f3f6eae9cba51e Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 16:53:16 -0800 Subject: [PATCH 1/7] Revert "Normalization: handle non-object top-level schemas; treat binary data as string (#22165)" This reverts commit 8276d033596af85ed0ec648d86fa23cbedbb01a9. --- .../seed/destination_definitions.yaml | 18 +++++++++--------- .../bases/base-normalization/Dockerfile | 2 +- .../transform_catalog/catalog_processor.py | 12 +----------- .../normalization/transform_catalog/utils.py | 7 ++----- .../basic-normalization.md | 1 - 5 files changed, 13 insertions(+), 27 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 4b8a24d6daf8..e606948b0804 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -45,7 +45,7 @@ icon: bigquery.svg normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: bigquery supportsDbt: true resourceRequirements: @@ -91,7 +91,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: clickhouse supportsDbt: true - name: Cloudflare R2 @@ -213,7 +213,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mssql - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: mssql supportsDbt: true - name: MeiliSearch @@ -239,7 +239,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mysql - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: mysql supportsDbt: true - name: Oracle @@ -251,7 +251,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-oracle - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: oracle supportsDbt: true - name: Postgres @@ -263,7 +263,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: postgres supportsDbt: true - name: Pulsar @@ -295,7 +295,7 @@ icon: redshift.svg normalizationConfig: normalizationRepository: airbyte/normalization-redshift - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: redshift supportsDbt: true resourceRequirements: @@ -353,7 +353,7 @@ icon: snowflake.svg normalizationConfig: normalizationRepository: airbyte/normalization-snowflake - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: snowflake supportsDbt: true resourceRequirements: @@ -407,7 +407,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-tidb - normalizationTag: 0.3.2 + normalizationTag: 0.3.1 normalizationIntegrationType: tidb supportsDbt: true - name: Typesense diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 098d477e6445..3ba6bc511292 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.3.2 +LABEL io.airbyte.version=0.3.1 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py index 4e4438f37783..2cbfe09394e4 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/catalog_processor.py @@ -135,17 +135,7 @@ def build_stream_processor( primary_key = get_field(configured_stream, "primary_key", f"Undefined primary key for stream {stream_name}") message = f"'json_schema'.'properties' are not defined for stream {stream_name}" - stream_schema = get_field(stream_config, "json_schema", f"'json_schema' is not defined for stream {stream_name}") - if "properties" in stream_schema: - properties = get_field(stream_schema, "properties", message) - elif "oneOf" in stream_schema: - options = list(filter(lambda option: "properties" in option, stream_schema["oneOf"])) - if len(options) == 0: - raise KeyError(f"Stream {stream_name} does not have any properties") - # If there are multiple oneOf options, just pick the first one - we don't really support oneOf to begin with - properties = options[0]["properties"] - else: - raise KeyError(f"Stream {stream_name} does not have any properties and no oneOf option with properties") + properties = get_field(get_field(stream_config, "json_schema", message), "properties", message) from_table = dbt_macro.Source(schema_name, raw_table_name) diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py index 049bc23ade50..ec63b2705d9c 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py @@ -31,14 +31,11 @@ def is_reftype(definition: dict) -> bool: def is_string(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.STRING_TYPE)) or is_type_included( - definition, get_reftype_function(data_type.BINARY_DATA_TYPE) - ) + return is_type_included(definition, get_reftype_function(data_type.STRING_TYPE)) def is_binary_datatype(definition: dict) -> bool: - return False - # return is_type_included(definition, get_reftype_function(data_type.BINARY_DATA_TYPE)) + return is_type_included(definition, get_reftype_function(data_type.BINARY_DATA_TYPE)) def is_datetime(definition: dict) -> bool: diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 2671c72d2378..e5655c6218dd 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,7 +353,6 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------|:----------------------|:-----------|:-------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| -| | 0.3.2 | 2023-01-31 | [\#22165](https://github.com/airbytehq/airbyte/pull/22165) | Fix support for non-object top-level schemas | | | 0.3.1 | 2023-01-31 | [\#22161](https://github.com/airbytehq/airbyte/pull/22161) | Fix handling for combined primitive types | | | 0.3.0 | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721) | Update normalization to airbyte-protocol v1.0.0 | | | 0.2.25 | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update Clickhouse dbt version | From 804eeaf01d3ebacc59e9b7dc480f5fbb4f65c472 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 16:53:29 -0800 Subject: [PATCH 2/7] Revert "Normalization: check for ref type existence (#22161)" This reverts commit dbe56d6fc28a1a2eb18e4514033bf2bd91174f57. --- .../seed/destination_definitions.yaml | 18 +++++++++--------- .../bases/base-normalization/Dockerfile | 2 +- .../normalization/transform_catalog/utils.py | 3 +-- .../basic-normalization.md | 1 - 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index e606948b0804..49044b747ff7 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -45,7 +45,7 @@ icon: bigquery.svg normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: bigquery supportsDbt: true resourceRequirements: @@ -91,7 +91,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: clickhouse supportsDbt: true - name: Cloudflare R2 @@ -213,7 +213,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mssql - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: mssql supportsDbt: true - name: MeiliSearch @@ -239,7 +239,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mysql - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: mysql supportsDbt: true - name: Oracle @@ -251,7 +251,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-oracle - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: oracle supportsDbt: true - name: Postgres @@ -263,7 +263,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: postgres supportsDbt: true - name: Pulsar @@ -295,7 +295,7 @@ icon: redshift.svg normalizationConfig: normalizationRepository: airbyte/normalization-redshift - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: redshift supportsDbt: true resourceRequirements: @@ -353,7 +353,7 @@ icon: snowflake.svg normalizationConfig: normalizationRepository: airbyte/normalization-snowflake - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: snowflake supportsDbt: true resourceRequirements: @@ -407,7 +407,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-tidb - normalizationTag: 0.3.1 + normalizationTag: 0.3.0 normalizationIntegrationType: tidb supportsDbt: true - name: Typesense diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 3ba6bc511292..eb985d00e88d 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.3.1 +LABEL io.airbyte.version=0.3.0 LABEL io.airbyte.name=airbyte/normalization diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py index ec63b2705d9c..10d6ee35af14 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py @@ -118,8 +118,7 @@ def is_simple_property(definition: dict) -> bool: def is_combining_node(properties: dict) -> Set[str]: # this case appears when we have analog of old protocol like id: {type:[number, string]} and it's handled separately if data_type.ONE_OF_VAR_NAME in properties and any( - data_type.REF_TYPE_VAR_NAME in option and data_type.WELL_KNOWN_TYPE_VAR_NAME in option[data_type.REF_TYPE_VAR_NAME] - for option in properties[data_type.ONE_OF_VAR_NAME] + data_type.WELL_KNOWN_TYPE_VAR_NAME in option[data_type.REF_TYPE_VAR_NAME] for option in properties[data_type.ONE_OF_VAR_NAME] ): return set() else: diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index e5655c6218dd..876b14edded2 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,7 +353,6 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------|:----------------------|:-----------|:-------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| -| | 0.3.1 | 2023-01-31 | [\#22161](https://github.com/airbytehq/airbyte/pull/22161) | Fix handling for combined primitive types | | | 0.3.0 | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721) | Update normalization to airbyte-protocol v1.0.0 | | | 0.2.25 | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update Clickhouse dbt version | | | 0.2.24 | 2022-11-01 | [\#18015](https://github.com/airbytehq/airbyte/pull/18015) | Add a drop table hook that drops *_scd tables after overwrite/reset | From 4d20c86c130e4ea37c98b5446b952307359a7dc0 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 16:53:40 -0800 Subject: [PATCH 3/7] =?UTF-8?q?Revert=20"=F0=9F=8E=89Updated=20normalizati?= =?UTF-8?q?on=20to=20handle=20new=20datatypes=20(#19721)"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit c1d77366392c9004f30d06da68b035d205ddcb25. --- .../macros/cross_db_utils/datatypes.sql | 30 --- ..._columns_resulting_into_long_names_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 5 - .../dedup_exchange_rate_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../dedup_exchange_rate_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 16 -- .../test_normalization/exchange_rate.sql | 5 - .../dedup_exchange_rate_ab1.sql | 2 +- .../dedup_exchange_rate_ab2.sql | 10 +- .../dedup_cdc_excluded_scd.sql | 6 +- .../dedup_exchange_rate_scd.sql | 6 +- .../renamed_dedup_cdc_excluded_scd.sql | 6 +- .../dedup_exchange_rate.sql | 6 +- .../renamed_dedup_cdc_excluded.sql | 6 +- .../test_normalization/exchange_rate.sql | 5 +- .../dedup_exchange_rate_stg.sql | 2 +- .../multiple_column_names_conflicts_stg.sql | 2 +- .../dedup_exchange_rate_ab2.sql | 8 +- .../renamed_dedup_cdc_excluded_ab2.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../dedup_exchange_rate_ab2.sql | 8 +- .../dedup_exchange_rate_scd.sql | 7 +- .../renamed_dedup_cdc_excluded_scd.sql | 7 +- .../dedup_exchange_rate.sql | 7 +- .../renamed_dedup_cdc_excluded.sql | 7 +- .../test_normalization/exchange_rate.sql | 5 +- .../dedup_exchange_rate_stg.sql | 2 +- .../mssql/test_nested_streams/dbt_project.yml | 138 ++++---------- ...eam_with_co__lting_into_long_names_scd.sql | 4 +- .../models/generated/sources.yml | 1 - .../mssql/test_simple_streams/dbt_project.yml | 103 ++++------ .../test_normalization/exchange_rate.sql | 4 - .../dedup_exchange_rate_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../models/generated/sources.yml | 1 - .../test_normalization/exchange_rate.sql | 4 - ..._stream_with_co_1g_into_long_names_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 50 +++-- .../dedup_exchange_rate_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../test_normalization/exchange_rate.sql | 50 +++-- .../test_normalization/exchange_rate.sql | 8 +- .../dedup_exchange_rate_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../test_normalization/exchange_rate.sql | 8 +- ...ream_with_c__lting_into_long_names_scd.sql | 4 +- .../some_stream_that_was_empty_scd.sql | 4 +- .../test_normalization/arrays.sql | 16 -- .../arrays_nested_array_parent.sql | 16 -- .../conflict_stream_array.sql | 16 -- .../conflict_stream_name.sql | 16 -- ...ict_stream_name___conflict_stream_name.sql | 16 -- ...flict_stream_name_conflict_stream_name.sql | 16 -- .../conflict_stream_scalar.sql | 16 -- ...ested_stream_wi__lting_into_long_names.sql | 16 -- .../test_normalization/unnest_alias.sql | 16 -- ...t_alias_childre__column___with__quotes.sql | 16 -- .../unnest_alias_children.sql | 16 -- .../unnest_alias_children_owner.sql | 16 -- .../test_normalization/exchange_rate.sql | 5 +- .../test_normalization/exchange_rate_ab1.sql | 1 - .../test_normalization/exchange_rate_ab2.sql | 1 - .../test_normalization/exchange_rate_ab3.sql | 1 - .../1_prefix_startwith_number_scd.sql | 4 +- .../dedup_cdc_excluded_scd.sql | 4 +- .../dedup_exchange_rate_scd.sql | 4 +- .../multiple_column_names_conflicts_scd.sql | 4 +- .../test_normalization/pos_dedup_cdcx_scd.sql | 4 +- .../renamed_dedup_cdc_excluded_scd.sql | 4 +- .../test_normalization/types_testing_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 17 -- .../dedup_cdc_excluded_scd.sql | 4 +- .../dedup_exchange_rate_scd.sql | 4 +- .../renamed_dedup_cdc_excluded_scd.sql | 4 +- .../test_normalization/exchange_rate.sql | 16 -- .../test_normalization/exchange_rate.sql | 5 +- .../test_nested_streams/dbt_project.yml | 140 +++++++------- ..._columns_resulting_into_long_names_scd.sql | 6 +- ..._columns_resulting_into_long_names_scd.sql | 15 -- ...plex_columns_resulting_into_long_names.sql | 29 +++ ...ns_resulting_into_long_names_partition.sql | 8 +- ...sulting_into_long_names_partition_data.sql | 10 +- ...long_names_partition_double_array_data.sql | 10 +- ...plex_columns_resulting_into_long_names.sql | 15 -- ...ns_resulting_into_long_names_partition.sql | 9 - ...sulting_into_long_names_partition_data.sql | 9 - ...long_names_partition_double_array_data.sql | 9 - ...plex_columns_resulting_into_long_names.sql | 29 --- ..._columns_resulting_into_long_names_ab1.sql | 6 +- ..._columns_resulting_into_long_names_ab2.sql | 2 +- ...esulting_into_long_names_partition_ab1.sql | 2 +- ...ing_into_long_names_partition_data_ab1.sql | 2 +- ..._names_partition_double_array_data_ab1.sql | 2 +- ..._columns_resulting_into_long_names_scd.sql | 12 +- ...plex_columns_resulting_into_long_names.sql | 4 +- ...ns_resulting_into_long_names_partition.sql | 2 +- ...sulting_into_long_names_partition_data.sql | 2 +- ...long_names_partition_double_array_data.sql | 2 +- .../models/generated/sources.yml | 16 +- ..._columns_resulting_into_long_names_scd.sql | 4 +- ..._columns_resulting_into_long_names_scd.sql | 15 -- ...plex_columns_resulting_into_long_names.sql | 4 +- ...ns_resulting_into_long_names_partition.sql | 9 + ...sulting_into_long_names_partition_data.sql | 9 + ...long_names_partition_double_array_data.sql | 9 + ...ns_resulting_into_long_names_partition.sql | 9 - ...sulting_into_long_names_partition_data.sql | 9 - ...long_names_partition_double_array_data.sql | 9 - ...plex_columns_resulting_into_long_names.sql | 15 -- ...ns_resulting_into_long_names_partition.sql | 9 - ...sulting_into_long_names_partition_data.sql | 9 - ...long_names_partition_double_array_data.sql | 9 - .../test_simple_streams/dbt_project.yml | 38 ++-- .../test_simple_streams/first_dbt_project.yml | 78 ++++---- .../dedup_exchange_rate_scd.sql | 6 +- .../dedup_exchange_rate_scd.sql | 15 -- .../dedup_exchange_rate.sql | 8 +- .../dedup_exchange_rate.sql | 15 -- .../test_normalization}/exchange_rate.sql | 22 +-- .../exchange_rate.sql | 115 ------------ .../exchange_rate.sql | 115 ------------ .../dedup_exchange_rate_stg.sql | 8 +- .../multiple_column_names_conflicts_stg.sql | 6 +- .../multiple_column_names_conflicts_stg.sql | 62 ------ .../dedup_exchange_rate_stg.sql | 66 ------- .../dedup_exchange_rate_ab1.sql | 6 +- .../dedup_exchange_rate_ab2.sql | 2 +- .../dedup_exchange_rate_scd.sql | 12 +- .../dedup_exchange_rate.sql | 4 +- .../test_normalization/exchange_rate.sql | 26 +++ .../exchange_rate.sql | 47 ----- .../dedup_exchange_rate_stg.sql | 2 +- .../models/generated/sources.yml | 2 +- .../dedup_exchange_rate_ab1.sql | 6 +- .../dedup_exchange_rate_ab2.sql | 2 +- .../dedup_exchange_rate_scd.sql | 12 +- .../dedup_exchange_rate.sql | 4 +- .../test_normalization/exchange_rate.sql | 26 +++ .../exchange_rate.sql | 42 ----- .../dedup_exchange_rate_stg.sql | 2 +- .../modified_models/generated/sources.yml | 2 +- .../dedup_exchange_rate_scd.sql | 4 +- .../dedup_exchange_rate_scd.sql | 15 -- .../dedup_exchange_rate.sql | 4 +- .../dedup_exchange_rate.sql | 15 -- .../exchange_rate.sql | 35 +--- .../exchange_rate.sql | 115 ------------ .../dedup_exchange_rate_stg.sql | 8 +- .../dedup_exchange_rate_stg.sql | 66 ------- .../dedup_exchange_rate_scd.sql | 4 +- .../dedup_exchange_rate_scd.sql | 15 -- .../dedup_exchange_rate.sql | 4 +- .../dedup_exchange_rate.sql | 15 -- .../exchange_rate.sql | 10 +- .../dedup_exchange_rate_stg.sql | 8 +- .../dedup_exchange_rate_stg.sql | 68 ------- ..._COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql | 4 +- .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 5 - .../DEDUP_EXCHANGE_RATE_SCD.sql | 4 +- .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 17 -- .../TEST_NORMALIZATION/EXCHANGE_RATE.sql | 5 - .../tidb/test_nested_streams/dbt_project.yml | 121 ------------ ..._stream_with_co_1g_into_long_names_scd.sql | 20 -- ..._stream_with_co___long_names_partition.sql | 20 -- ..._stream_with_co___names_partition_data.sql | 20 -- ..._stream_with_co__ion_double_array_data.sql | 20 -- ..._stream_with_co__lting_into_long_names.sql | 20 -- ..._stream_with_co_1g_into_long_names_ab1.sql | 19 -- ..._stream_with_co_1g_into_long_names_ab2.sql | 19 -- ..._stream_with_co_2g_names_partition_ab1.sql | 19 -- ..._stream_with_co_3double_array_data_ab1.sql | 20 -- ..._stream_with_co_3es_partition_data_ab1.sql | 20 -- ..._stream_with_co_1g_into_long_names_scd.sql | 162 ---------------- ..._stream_with_co___long_names_partition.sql | 19 -- ..._stream_with_co___names_partition_data.sql | 18 -- ..._stream_with_co__ion_double_array_data.sql | 18 -- ..._stream_with_co__lting_into_long_names.sql | 22 --- .../models/generated/sources.yml | 23 --- ..._stream_with_co_1g_into_long_names_scd.sql | 7 - ..._stream_with_co___long_names_partition.sql | 7 - ..._stream_with_co___names_partition_data.sql | 7 - ..._stream_with_co__ion_double_array_data.sql | 7 - ..._stream_with_co__lting_into_long_names.sql | 7 - .../tidb/test_simple_streams/dbt_project.yml | 86 --------- .../dedup_exchange_rate_scd.sql | 20 -- .../dedup_exchange_rate.sql | 20 -- .../test_normalization/exchange_rate.sql | 17 -- .../dedup_exchange_rate_stg.sql | 112 ----------- .../multiple_column_names_conflicts_stg.sql | 103 ---------- .../dedup_exchange_rate_ab1.sql | 24 --- .../dedup_exchange_rate_ab2.sql | 27 --- .../dedup_exchange_rate_scd.sql | 176 ------------------ .../dedup_exchange_rate.sql | 27 --- .../test_normalization/exchange_rate.sql | 46 ----- .../dedup_exchange_rate_stg.sql | 24 --- .../models/generated/sources.yml | 16 -- .../dedup_exchange_rate_scd.sql | 7 - .../dedup_exchange_rate.sql | 7 - .../test_normalization/exchange_rate.sql | 17 -- .../dedup_exchange_rate_stg.sql | 112 ----------- .../destination_catalog.json | 14 +- .../messages1.txt | 8 +- .../messages2.txt | 8 +- .../data_input/catalog.json | 69 +++---- .../data_input/messages.txt | 22 +-- .../data_input/messages_incremental.txt | 28 +-- .../data_input/test_drop_scd_catalog.json | 18 +- .../test_drop_scd_catalog_incremental.json | 18 +- .../test_drop_scd_catalog_reset.json | 18 +- .../data_input/test_drop_scd_messages.txt | 10 +- .../test_scd_reset_messages_incremental.txt | 12 +- .../data_input/catalog.json | 115 ++++++------ .../data_input/catalog_schema_change.json | 58 +++--- .../data_input/messages.txt | 104 +++++------ .../data_input/messages_incremental.txt | 42 ++--- .../data_input/messages_schema_change.txt | 24 +-- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../types_testing_binary_values.sql | 14 -- .../integration_tests/test_ephemeral.py | 4 +- .../integration_tests/test_normalization.py | 79 +------- .../normalization/data_type.py | 20 -- .../transform_catalog/stream_processor.py | 144 ++++---------- .../normalization/transform_catalog/utils.py | 101 +++++----- ...long_name_truncate_collisions_catalog.json | 6 +- .../unit_tests/resources/nested_catalog.json | 138 +++++++------- .../un-nesting_collisions_catalog.json | 10 +- .../unit_tests/test_stream_processor.py | 16 +- 234 files changed, 1043 insertions(+), 4175 deletions(-) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql (85%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql (78%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql (78%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql (83%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql (94%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql (92%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql (94%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/{test_normalization_iprwf => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/{test_normalization_fxlbt => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql (50%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/{test_normalization_fxlbt => test_normalization}/nested_stream_with_complex_columns_resulting_into_long_names.sql (50%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_scd.sql (89%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/{test_normalization_spffv => test_normalization}/dedup_exchange_rate.sql (58%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/{third_output/airbyte_tables/test_normalization_vorny => first_output/airbyte_tables/test_normalization}/exchange_rate.sql (71%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_stg.sql (88%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/{test_normalization_vorny => test_normalization}/multiple_column_names_conflicts_stg.sql (87%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/multiple_column_names_conflicts_stg.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_ab1.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_ab2.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_scd.sql (92%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/{test_normalization_spffv => test_normalization}/dedup_exchange_rate.sql (87%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_stg.sql (93%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_ab1.sql (87%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_ab2.sql (95%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_scd.sql (92%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/{test_normalization_spffv => test_normalization}/dedup_exchange_rate.sql (87%) create mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_stg.sql (93%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_scd.sql (52%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/{test_normalization_vorny => test_normalization}/dedup_exchange_rate.sql (52%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/{test_normalization_spffv => test_normalization}/exchange_rate.sql (66%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_stg.sql (88%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/{test_normalization_vorny => test_normalization}/dedup_exchange_rate_scd.sql (52%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/{test_normalization_spffv => test_normalization}/dedup_exchange_rate.sql (52%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/{test_normalization_spffv => test_normalization}/exchange_rate.sql (89%) rename airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/{test_normalization_spffv => test_normalization}/dedup_exchange_rate_stg.sql (88%) delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql delete mode 100755 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/dbt_project.yml delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/sources.yml delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql delete mode 100755 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/dbt_project.yml delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/sources.yml delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_bigquery_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_clickhouse_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mssql_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mysql_tidb_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_oracle_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_postgres_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_redshift_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_snowflake_tmp/types_testing_binary_values.sql delete mode 100644 airbyte-integrations/bases/base-normalization/normalization/data_type.py diff --git a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql index f868727464bd..42f5312b054f 100755 --- a/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql +++ b/airbyte-integrations/bases/base-normalization/dbt-project-template/macros/cross_db_utils/datatypes.sql @@ -8,14 +8,6 @@ string {% endmacro %} -{%- macro type_binary() -%} - {{ adapter.dispatch('type_binary')() }} -{%- endmacro -%} - -{%- macro default__type_binary() -%} - binary -{%- endmacro -%} - {%- macro redshift__type_json() -%} {%- if redshift_super_type() -%} super @@ -80,28 +72,6 @@ char(1000) {%- endmacro -%} -{# binary data ------------------------------------------------- #} - -{%- macro postgres__type_binary() -%} - bytea -{%- endmacro -%} - -{%- macro bigquery__type_binary() -%} - bytes -{%- endmacro -%} - -{%- macro mssql__type_binary() -%} - VARBINARY(MAX) -{%- endmacro -%} - -{%- macro snowflake__type_binary() -%} - VARBINARY -{%- endmacro -%} - -{%- macro clickhouse__type_binary() -%} - VARBINARY -{%- endmacro -%} - {# float ------------------------------------------------- #} {% macro mysql__type_float() %} float diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index a94acefbf5cc..1df163184ca0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -34,12 +34,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 9309b066c3a9..3d32bbb2838a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -24,7 +24,6 @@ select json_extract_scalar(_airbyte_data, "$['datetime_no_tz']") as datetime_no_tz, json_extract_scalar(_airbyte_data, "$['time_tz']") as time_tz, json_extract_scalar(_airbyte_data, "$['time_no_tz']") as time_no_tz, - json_extract_scalar(_airbyte_data, "$['property_binary_data']") as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at @@ -75,7 +74,6 @@ select cast(nullif(time_no_tz, '') as time ) as time_no_tz, - cast(FROM_BASE64(property_binary_data) as bytes) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at @@ -113,8 +111,6 @@ select string ), ''), '-', coalesce(cast(time_no_tz as string -), ''), '-', coalesce(cast(property_binary_data as - string ), '')) as string ))) as _airbyte_exchange_rate_hashid, @@ -138,7 +134,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2d33e1331c8f..ce21bef8c722 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -34,12 +34,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index 0cb9f41b4faf..61b42d20863c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -3,22 +3,6 @@ partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -37,7 +21,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 6dca5a736c9a..4f6b80934992 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -34,12 +34,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql index b9817a4de3a0..84cb4985e8c9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -3,22 +3,6 @@ partition_by = {"field": "_airbyte_emitted_at", "data_type": "timestamp", "granularity": "day"}, unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index 9309b066c3a9..3d32bbb2838a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -24,7 +24,6 @@ select json_extract_scalar(_airbyte_data, "$['datetime_no_tz']") as datetime_no_tz, json_extract_scalar(_airbyte_data, "$['time_tz']") as time_tz, json_extract_scalar(_airbyte_data, "$['time_no_tz']") as time_no_tz, - json_extract_scalar(_airbyte_data, "$['property_binary_data']") as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at @@ -75,7 +74,6 @@ select cast(nullif(time_no_tz, '') as time ) as time_no_tz, - cast(FROM_BASE64(property_binary_data) as bytes) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at @@ -113,8 +111,6 @@ select string ), ''), '-', coalesce(cast(time_no_tz as string -), ''), '-', coalesce(cast(property_binary_data as - string ), '')) as string ))) as _airbyte_exchange_rate_hashid, @@ -138,7 +134,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, CURRENT_TIMESTAMP() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 1cee42ac010e..2609c12f32d3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,6 +1,6 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_ab1 + create view _airbyte_test_normalization.dedup_exchange_rate_ab1__dbt_tmp as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 1a5e6ad10142..07778080d6fa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,26 +1,26 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_ab2 + create view _airbyte_test_normalization.dedup_exchange_rate_ab2__dbt_tmp as ( -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: _airbyte_test_normalization.dedup_exchange_rate_ab1 select - accurateCastOrNull(trim(BOTH '"' from id), ' + accurateCastOrNull(id, ' BIGINT ') as id, nullif(accurateCastOrNull(trim(BOTH '"' from currency), 'String'), 'null') as currency, toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from nullif(date, '')))) as date, parseDateTime64BestEffortOrNull(trim(BOTH '"' from nullif(timestamp_col, ''))) as timestamp_col, - accurateCastOrNull(trim(BOTH '"' from "HKD@spéçiäl & characters"), ' + accurateCastOrNull("HKD@spéçiäl & characters", ' Float64 ') as "HKD@spéçiäl & characters", nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), 'String'), 'null') as HKD_special___characters, - accurateCastOrNull(trim(BOTH '"' from NZD), ' + accurateCastOrNull(NZD, ' Float64 ') as NZD, - accurateCastOrNull(trim(BOTH '"' from USD), ' + accurateCastOrNull(USD, ' Float64 ') as USD, _airbyte_ab_id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index 654648c600c6..7dac7b7d793f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -1,7 +1,5 @@ - - - + insert into test_normalization.dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "name", "_ab_cdc_lsn", "_ab_cdc_updated_at", "_ab_cdc_deleted_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_cdc_excluded_hashid") @@ -103,4 +101,4 @@ select _airbyte_dedup_cdc_excluded_hashid from dedup_data where _airbyte_row_num = 1 - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 57682bc247c3..a3527b053dc3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,5 @@ - - - + insert into test_normalization.dedup_exchange_rate_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") @@ -107,4 +105,4 @@ select _airbyte_dedup_exchange_rate_hashid from dedup_data where _airbyte_row_num = 1 - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index f56e43bdf65b..cf48610f8b82 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -1,7 +1,5 @@ - - - + insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") @@ -89,4 +87,4 @@ select _airbyte_renamed_dedup_cdc_excluded_hashid from dedup_data where _airbyte_row_num = 1 - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index e2340bdfe303..11d81fef34b9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,7 +1,5 @@ - - - + insert into test_normalization.dedup_exchange_rate ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") @@ -28,4 +26,4 @@ where 1 = 1 and _airbyte_active_row = 1 - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index daf0b9959568..b237171bc7fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -1,7 +1,5 @@ - - - + insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") @@ -22,4 +20,4 @@ where 1 = 1 and _airbyte_active_row = 1 - \ No newline at end of file + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 46e79d1d45cb..c2be71e63fc9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,9 +1,7 @@ - - - insert into test_normalization.exchange_rate ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "property_binary_data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + insert into test_normalization.exchange_rate__dbt_tmp ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") -- Final base SQL model -- depends_on: _airbyte_test_normalization.exchange_rate_ab3 @@ -21,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 98084e148f4a..9a932053975b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,6 +1,6 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_stg + create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql index 57a8d9b9532d..5f1062999579 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql @@ -1,6 +1,6 @@ - create view _airbyte_test_normalization.multiple_column_names_conflicts_stg + create view _airbyte_test_normalization.multiple_column_names_conflicts_stg__dbt_tmp as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index fb6b3673aaa2..22f82153a5cd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -6,14 +6,14 @@ -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: {{ ref('dedup_exchange_rate_ab1') }} select - accurateCastOrNull(trim(BOTH '"' from id), '{{ dbt_utils.type_bigint() }}') as id, + accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, nullif(accurateCastOrNull(trim(BOTH '"' from currency), '{{ dbt_utils.type_string() }}'), 'null') as currency, toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from {{ empty_string_to_null('date') }}))) as date, parseDateTime64BestEffortOrNull(trim(BOTH '"' from {{ empty_string_to_null('timestamp_col') }})) as timestamp_col, - accurateCastOrNull(trim(BOTH '"' from {{ quote('HKD@spéçiäl & characters') }}), '{{ dbt_utils.type_float() }}') as {{ quote('HKD@spéçiäl & characters') }}, + accurateCastOrNull({{ quote('HKD@spéçiäl & characters') }}, '{{ dbt_utils.type_float() }}') as {{ quote('HKD@spéçiäl & characters') }}, nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), '{{ dbt_utils.type_string() }}'), 'null') as HKD_special___characters, - accurateCastOrNull(trim(BOTH '"' from NZD), '{{ dbt_utils.type_float() }}') as NZD, - accurateCastOrNull(trim(BOTH '"' from USD), '{{ dbt_utils.type_float() }}') as USD, + accurateCastOrNull(NZD, '{{ dbt_utils.type_float() }}') as NZD, + accurateCastOrNull(USD, '{{ dbt_utils.type_float() }}') as USD, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql index 211ba2da4a46..c6885e98962e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_ctes/test_normalization/renamed_dedup_cdc_excluded_ab2.sql @@ -6,8 +6,8 @@ -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: {{ ref('renamed_dedup_cdc_excluded_ab1') }} select - accurateCastOrNull(trim(BOTH '"' from id), '{{ dbt_utils.type_bigint() }}') as id, - accurateCastOrNull(trim(BOTH '"' from _ab_cdc_updated_at), '{{ dbt_utils.type_float() }}') as _ab_cdc_updated_at, + accurateCastOrNull(id, '{{ dbt_utils.type_bigint() }}') as id, + accurateCastOrNull(_ab_cdc_updated_at, '{{ dbt_utils.type_float() }}') as _ab_cdc_updated_at, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index 1bd94c36e50f..c66443b3a150 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,22 +1,6 @@ {{ config( unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -35,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 5bad46684b29..07778080d6fa 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -7,20 +7,20 @@ -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: _airbyte_test_normalization.dedup_exchange_rate_ab1 select - accurateCastOrNull(trim(BOTH '"' from id), ' + accurateCastOrNull(id, ' BIGINT ') as id, nullif(accurateCastOrNull(trim(BOTH '"' from currency), 'String'), 'null') as currency, toDate(parseDateTimeBestEffortOrNull(trim(BOTH '"' from nullif(date, '')))) as date, parseDateTime64BestEffortOrNull(trim(BOTH '"' from nullif(timestamp_col, ''))) as timestamp_col, - accurateCastOrNull(trim(BOTH '"' from "HKD@spéçiäl & characters"), ' + accurateCastOrNull("HKD@spéçiäl & characters", ' Float64 ') as "HKD@spéçiäl & characters", nullif(accurateCastOrNull(trim(BOTH '"' from HKD_special___characters), 'String'), 'null') as HKD_special___characters, - accurateCastOrNull(trim(BOTH '"' from NZD), ' + accurateCastOrNull(NZD, ' Float64 ') as NZD, - accurateCastOrNull(trim(BOTH '"' from USD), ' + accurateCastOrNull(USD, ' Float64 ') as USD, _airbyte_ab_id, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2d4ac4e196ea..a793d7412e48 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,3 +1,6 @@ - create table test_normalization.dedup_exchange_rate_scd__dbt_tmp as test_normalization.dedup_exchange_rate_scd__dbt_new_data - \ No newline at end of file + insert into test_normalization.dedup_exchange_rate_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from dedup_exchange_rate_scd__dbt_tmp + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 59a6053de99c..8f84c4f3c162 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -1,3 +1,6 @@ - create table test_normalization.renamed_dedup_cdc_excluded_scd__dbt_tmp as test_normalization.renamed_dedup_cdc_excluded_scd__dbt_new_data - \ No newline at end of file + insert into test_normalization.renamed_dedup_cdc_excluded_scd ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "_ab_cdc_updated_at", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from renamed_dedup_cdc_excluded_scd__dbt_tmp + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index e34e47816341..4a895d6cf480 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,3 +1,6 @@ - create table test_normalization.dedup_exchange_rate__dbt_tmp as test_normalization.dedup_exchange_rate__dbt_new_data - \ No newline at end of file + insert into test_normalization.dedup_exchange_rate ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" + from dedup_exchange_rate__dbt_tmp + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql index 5d6352d6aa6d..1b96d3f87152 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_incremental/test_normalization/renamed_dedup_cdc_excluded.sql @@ -1,3 +1,6 @@ - create table test_normalization.renamed_dedup_cdc_excluded__dbt_tmp as test_normalization.renamed_dedup_cdc_excluded__dbt_new_data - \ No newline at end of file + insert into test_normalization.renamed_dedup_cdc_excluded ("_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid") + select "_airbyte_unique_key", "id", "_ab_cdc_updated_at", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_renamed_dedup_cdc_excluded_hashid" + from renamed_dedup_cdc_excluded__dbt_tmp + + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index a8075c3f95bb..c2be71e63fc9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,9 +1,7 @@ - - - insert into test_normalization.exchange_rate__dbt_backup ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "property_binary_data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") + insert into test_normalization.exchange_rate__dbt_tmp ("id", "currency", "date", "timestamp_col", "HKD@spéçiäl & characters", "HKD_special___characters", "NZD", "USD", "column___with__quotes", "datetime_tz", "datetime_no_tz", "time_tz", "time_no_tz", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_exchange_rate_hashid") -- Final base SQL model -- depends_on: _airbyte_test_normalization.exchange_rate_ab3 @@ -21,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 98084e148f4a..9a932053975b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,6 +1,6 @@ - create view _airbyte_test_normalization.dedup_exchange_rate_stg + create view _airbyte_test_normalization.dedup_exchange_rate_stg__dbt_tmp as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml index 9e5be22d283b..8ed082f36774 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/dbt_project.yml @@ -1,29 +1,45 @@ -name: airbyte_utils -version: '1.0' +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "airbyte_utils" +version: "1.0" config-version: 2 -profile: normalize -model-paths: -- models -docs-paths: -- docs -analysis-paths: -- analysis -test-paths: -- tests -seed-paths: -- data -macro-paths: -- macros -target-path: ../build -log-path: ../logs -packages-install-path: /dbt -clean-targets: -- build -- dbt_modules + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: "normalize" + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "/dbt" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + quoting: database: true + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true + +# You can define configurations for models in the `model-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! models: airbyte_utils: +materialized: table @@ -40,82 +56,6 @@ models: airbyte_views: +tags: airbyte_internal_views +materialized: view + vars: - dbt_utils_dispatch_list: - - airbyte_utils - json_column: _airbyte_data - models_to_source: - nested_stream_with_co__lting_into_long_names_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__lting_into_long_names_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__lting_into_long_names_stg: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__lting_into_long_names_scd: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__lting_into_long_names: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - non_nested_stream_wit__lting_into_long_names_ab1: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_wit__lting_into_long_names_ab2: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_wit__lting_into_long_names_ab3: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_wit__lting_into_long_names: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - some_stream_that_was_empty_ab1: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_ab2: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_stg: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_scd: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty: test_normalization._airbyte_raw_some_stream_that_was_empty - simple_stream_with_na__lting_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - simple_stream_with_na__lting_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - simple_stream_with_na__lting_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - simple_stream_with_na__lting_into_long_names: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_scalar_ab1: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab2: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab3: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_array_ab1: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab2: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab3: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array: test_normalization._airbyte_raw_conflict_stream_array - unnest_alias_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias: test_normalization._airbyte_raw_unnest_alias - arrays_ab1: test_normalization._airbyte_raw_arrays - arrays_ab2: test_normalization._airbyte_raw_arrays - arrays_ab3: test_normalization._airbyte_raw_arrays - arrays: test_normalization._airbyte_raw_arrays - nested_stream_with_co___long_names_partition_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___long_names_partition_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___long_names_partition_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___long_names_partition: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name_conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children: test_normalization._airbyte_raw_unnest_alias - arrays_nested_array_parent_ab1: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent_ab2: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent_ab3: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent: test_normalization._airbyte_raw_arrays - nested_stream_with_co__ion_double_array_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__ion_double_array_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__ion_double_array_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co__ion_double_array_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___names_partition_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___names_partition_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___names_partition_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_co___names_partition_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name____conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name____conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name____conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name____conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_owner_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children__column___with__quotes_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children__column___with__quotes_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children__column___with__quotes_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children__column___with__quotes: test_normalization._airbyte_raw_unnest_alias + dbt_utils_dispatch_list: ["airbyte_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql index 33fe9d4856b1..b4683a3ea301 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co__lting_into_long_names_scd.sql @@ -32,12 +32,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/sources.yml index 29bae1b4b510..92fa4c9a2580 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_nested_streams/models/generated/sources.yml @@ -6,7 +6,6 @@ sources: schema: false identifier: false tables: - - name: _airbyte_raw_arrays - name: _airbyte_raw_conflict_stream_array - name: _airbyte_raw_conflict_stream_name - name: _airbyte_raw_conflict_stream_scalar diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml index 70f5208e5e5d..8ed082f36774 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/dbt_project.yml @@ -1,29 +1,45 @@ -name: airbyte_utils -version: '1.0' +# This file is necessary to install dbt-utils with dbt deps +# the content will be overwritten by the transform function + +# Name your package! Package names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: "airbyte_utils" +version: "1.0" config-version: 2 -profile: normalize -model-paths: -- models -docs-paths: -- docs -analysis-paths: -- analysis -test-paths: -- tests -seed-paths: -- data -macro-paths: -- macros -target-path: ../build -log-path: ../logs -packages-install-path: /dbt -clean-targets: -- build -- dbt_modules + +# This setting configures which "profile" dbt uses for this project. Profiles contain +# database connection information, and should be configured in the ~/.dbt/profiles.yml file +profile: "normalize" + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that source models can be found +# in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +docs-paths: ["docs"] +analysis-paths: ["analysis"] +test-paths: ["tests"] +seed-paths: ["data"] +macro-paths: ["macros"] + +target-path: "../build" # directory which will store compiled SQL files +log-path: "../logs" # directory which will store DBT logs +packages-install-path: "/dbt" # directory which will store external DBT dependencies + +clean-targets: # directories to be removed by `dbt clean` + - "build" + - "dbt_modules" + quoting: database: true + # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785) + # all schemas should be unquoted schema: false identifier: true + +# You can define configurations for models in the `model-paths` directory here. +# Using these configurations, you can enable or disable models, change how they +# are materialized, and more! models: airbyte_utils: +materialized: table @@ -40,47 +56,6 @@ models: airbyte_views: +tags: airbyte_internal_views +materialized: view + vars: - dbt_utils_dispatch_list: - - airbyte_utils - json_column: _airbyte_data - models_to_source: - exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization._airbyte_raw_exchange_rate - exchange_rate: test_normalization._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization._airbyte_raw_dedup_cdc_excluded - pos_dedup_cdcx_ab1: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_ab2: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_stg: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_scd: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx: test_normalization._airbyte_raw_pos_dedup_cdcx - _1_prefix_startwith_number_ab1: test_normalization._airbyte_raw_1_prefix_startwith_number - _1_prefix_startwith_number_ab2: test_normalization._airbyte_raw_1_prefix_startwith_number - _1_prefix_startwith_number_stg: test_normalization._airbyte_raw_1_prefix_startwith_number - _1_prefix_startwith_number_scd: test_normalization._airbyte_raw_1_prefix_startwith_number - _1_prefix_startwith_number: test_normalization._airbyte_raw_1_prefix_startwith_number - multiple_column_names_conflicts_ab1: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_ab2: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts - types_testing_ab1: test_normalization._airbyte_raw_types_testing - types_testing_ab2: test_normalization._airbyte_raw_types_testing - types_testing_stg: test_normalization._airbyte_raw_types_testing - types_testing_scd: test_normalization._airbyte_raw_types_testing - types_testing: test_normalization._airbyte_raw_types_testing + dbt_utils_dispatch_list: ["airbyte_utils"] diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 0532716f3b18..830e76c6f0ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -37,7 +37,6 @@ select json_value(_airbyte_data, ''$."datetime_no_tz"'') as datetime_no_tz, json_value(_airbyte_data, ''$."time_tz"'') as time_tz, json_value(_airbyte_data, ''$."time_no_tz"'') as time_no_tz, - json_value(_airbyte_data, ''$."property_binary_data"'') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at @@ -75,7 +74,6 @@ select cast(nullif(time_no_tz, '''') as time ) as time_no_tz, - CAST(property_binary_data as XML ).value(''.'',''binary'') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at @@ -104,7 +102,6 @@ select NVARCHAR(max)), ''''), ''-'', coalesce(cast(datetime_no_tz as NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_tz as NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_no_tz as - NVARCHAR(max)), ''''), ''-'', coalesce(cast(property_binary_data as NVARCHAR(max)), ''''),''''), '''') as NVARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, tmp.* @@ -127,7 +124,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 835580aa0a06..8d9648114261 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -32,12 +32,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index 1446314e40e6..8a74de4c1533 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,22 +1,6 @@ {{ config( unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -35,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml index f51802427655..97bf0d05cbd4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/models/generated/sources.yml @@ -13,4 +13,3 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded - - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index 0532716f3b18..830e76c6f0ef 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mssql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -37,7 +37,6 @@ select json_value(_airbyte_data, ''$."datetime_no_tz"'') as datetime_no_tz, json_value(_airbyte_data, ''$."time_tz"'') as time_tz, json_value(_airbyte_data, ''$."time_no_tz"'') as time_no_tz, - json_value(_airbyte_data, ''$."property_binary_data"'') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at @@ -75,7 +74,6 @@ select cast(nullif(time_no_tz, '''') as time ) as time_no_tz, - CAST(property_binary_data as XML ).value(''.'',''binary'') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at @@ -104,7 +102,6 @@ select NVARCHAR(max)), ''''), ''-'', coalesce(cast(datetime_no_tz as NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_tz as NVARCHAR(max)), ''''), ''-'', coalesce(cast(time_no_tz as - NVARCHAR(max)), ''''), ''-'', coalesce(cast(property_binary_data as NVARCHAR(max)), ''''),''''), '''') as NVARCHAR(max)), '''')), 2) as _airbyte_exchange_rate_hashid, tmp.* @@ -127,7 +124,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, SYSDATETIME() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql index f4b9f7bb2f07..9ffb6bd5558c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql @@ -32,12 +32,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index 200429b86c97..540fc0e7911f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -3,43 +3,41 @@ create table test_normalization.`exchange_rate__dbt_tmp` as ( - + with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_exchange_rate select - json_value(_airbyte_data, + json_value(_airbyte_data, '$."id"' RETURNING CHAR) as id, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."currency"' RETURNING CHAR) as currency, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."date"' RETURNING CHAR) as `date`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."timestamp_col"' RETURNING CHAR) as timestamp_col, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."NZD"' RETURNING CHAR) as nzd, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."USD"' RETURNING CHAR) as usd, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."datetime_tz"' RETURNING CHAR) as datetime_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."datetime_no_tz"' RETURNING CHAR) as datetime_no_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."time_tz"' RETURNING CHAR) as time_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."time_no_tz"' RETURNING CHAR) as time_no_tz, - json_value(_airbyte_data, - '$."property_binary_data"' RETURNING CHAR) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at from test_normalization._airbyte_raw_exchange_rate as table_alias @@ -50,7 +48,7 @@ where 1 = 1 -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: __dbt__cte__exchange_rate_ab1 select - cast(id as + cast(id as signed ) as id, cast(currency as char(1024)) as currency, @@ -59,14 +57,14 @@ select end as `date` , cast(nullif(timestamp_col, '') as char(1024)) as timestamp_col, - cast(`HKD@spéçiäl & characters` as + cast(`HKD@spéçiäl & characters` as float ) as `HKD@spéçiäl & characters`, cast(hkd_special___characters as char(1024)) as hkd_special___characters, - cast(nzd as + cast(nzd as float ) as nzd, - cast(usd as + cast(usd as float ) as usd, cast(`column__'with"_quotes` as char(1024)) as `column__'with"_quotes`, @@ -76,13 +74,12 @@ select end as datetime_no_tz , nullif(cast(time_tz as char(1024)), "") as time_tz, - nullif(cast(time_no_tz as + nullif(cast(time_no_tz as time ), "") as time_no_tz, - cast(FROM_BASE64(property_binary_data) as binary) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at from __dbt__cte__exchange_rate_ab1 @@ -93,7 +90,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), ''), '-', coalesce(cast(property_binary_data as char), '')) as char)) as _airbyte_exchange_rate_hashid, + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), '')) as char)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -114,10 +111,9 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 499ae700143f..b1c2af62e4bf 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -32,12 +32,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index bc56f1c5a3cc..3fe3205727b8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,22 +1,6 @@ {{ config( unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -35,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index 200429b86c97..540fc0e7911f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -3,43 +3,41 @@ create table test_normalization.`exchange_rate__dbt_tmp` as ( - + with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema -- depends_on: test_normalization._airbyte_raw_exchange_rate select - json_value(_airbyte_data, + json_value(_airbyte_data, '$."id"' RETURNING CHAR) as id, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."currency"' RETURNING CHAR) as currency, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."date"' RETURNING CHAR) as `date`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."timestamp_col"' RETURNING CHAR) as timestamp_col, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."NZD"' RETURNING CHAR) as nzd, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."USD"' RETURNING CHAR) as usd, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."datetime_tz"' RETURNING CHAR) as datetime_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."datetime_no_tz"' RETURNING CHAR) as datetime_no_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."time_tz"' RETURNING CHAR) as time_tz, - json_value(_airbyte_data, + json_value(_airbyte_data, '$."time_no_tz"' RETURNING CHAR) as time_no_tz, - json_value(_airbyte_data, - '$."property_binary_data"' RETURNING CHAR) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at from test_normalization._airbyte_raw_exchange_rate as table_alias @@ -50,7 +48,7 @@ where 1 = 1 -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type -- depends_on: __dbt__cte__exchange_rate_ab1 select - cast(id as + cast(id as signed ) as id, cast(currency as char(1024)) as currency, @@ -59,14 +57,14 @@ select end as `date` , cast(nullif(timestamp_col, '') as char(1024)) as timestamp_col, - cast(`HKD@spéçiäl & characters` as + cast(`HKD@spéçiäl & characters` as float ) as `HKD@spéçiäl & characters`, cast(hkd_special___characters as char(1024)) as hkd_special___characters, - cast(nzd as + cast(nzd as float ) as nzd, - cast(usd as + cast(usd as float ) as usd, cast(`column__'with"_quotes` as char(1024)) as `column__'with"_quotes`, @@ -76,13 +74,12 @@ select end as datetime_no_tz , nullif(cast(time_tz as char(1024)), "") as time_tz, - nullif(cast(time_no_tz as + nullif(cast(time_no_tz as time ), "") as time_no_tz, - cast(FROM_BASE64(property_binary_data) as binary) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at from __dbt__cte__exchange_rate_ab1 @@ -93,7 +90,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), ''), '-', coalesce(cast(property_binary_data as char), '')) as char)) as _airbyte_exchange_rate_hashid, + md5(cast(concat(coalesce(cast(id as char), ''), '-', coalesce(cast(currency as char), ''), '-', coalesce(cast(`date` as char), ''), '-', coalesce(cast(timestamp_col as char), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char), ''), '-', coalesce(cast(hkd_special___characters as char), ''), '-', coalesce(cast(nzd as char), ''), '-', coalesce(cast(usd as char), ''), '-', coalesce(cast(`column__'with"_quotes` as char), ''), '-', coalesce(cast(datetime_tz as char), ''), '-', coalesce(cast(datetime_no_tz as char), ''), '-', coalesce(cast(time_tz as char), ''), '-', coalesce(cast(time_no_tz as char), '')) as char)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -114,10 +111,9 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, - + CURRENT_TIMESTAMP as _airbyte_normalized_at, _airbyte_exchange_rate_hashid diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index ae518b148c97..4292befa848b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -22,7 +22,6 @@ select json_value("_AIRBYTE_DATA", '$."datetime_no_tz"') as datetime_no_tz, json_value("_AIRBYTE_DATA", '$."time_tz"') as time_tz, json_value("_AIRBYTE_DATA", '$."time_no_tz"') as time_no_tz, - json_value("_AIRBYTE_DATA", '$."property_binary_data"') as property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", @@ -69,7 +68,6 @@ select cast(nullif(time_no_tz, '') as varchar2(4000) ) as time_no_tz, - cast(property_binary_data as varchar2(4000)) as property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", @@ -121,10 +119,7 @@ select time_tz || '~' || - time_no_tz || '~' || - - - property_binary_data + time_no_tz ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", tmp.* @@ -147,7 +142,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2cc63b91fa44..9320dbc51f60 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -32,12 +32,12 @@ from ( select distinct {{ quote('_AIRBYTE_UNIQUE_KEY') }} as unique_key from {{ this }} - where 1=1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), quote(this.schema) + '.' + quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), this.schema + '.' + quote('dedup_exchange_rate')) }} ) recent_records left join ( select {{ quote('_AIRBYTE_UNIQUE_KEY') }} as unique_key, count({{ quote('_AIRBYTE_UNIQUE_KEY') }}) as active_count from {{ this }} - where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), quote(this.schema) + '.' + quote('dedup_exchange_rate')) }} + where {{ quote('_AIRBYTE_ACTIVE_ROW') }} = 1 {{ incremental_clause(quote('_AIRBYTE_NORMALIZED_AT'), this.schema + '.' + quote('dedup_exchange_rate')) }} group by {{ quote('_AIRBYTE_UNIQUE_KEY') }} ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index d1c13cf50a64..2fa506176467 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,22 +1,6 @@ {{ config( unique_key = quote('_AIRBYTE_AB_ID'), schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -35,7 +19,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, {{ quote('_AIRBYTE_AB_ID') }}, {{ quote('_AIRBYTE_EMITTED_AT') }}, {{ current_timestamp() }} as {{ quote('_AIRBYTE_NORMALIZED_AT') }}, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index ae518b148c97..4292befa848b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -22,7 +22,6 @@ select json_value("_AIRBYTE_DATA", '$."datetime_no_tz"') as datetime_no_tz, json_value("_AIRBYTE_DATA", '$."time_tz"') as time_tz, json_value("_AIRBYTE_DATA", '$."time_no_tz"') as time_no_tz, - json_value("_AIRBYTE_DATA", '$."property_binary_data"') as property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", @@ -69,7 +68,6 @@ select cast(nullif(time_no_tz, '') as varchar2(4000) ) as time_no_tz, - cast(property_binary_data as varchar2(4000)) as property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", @@ -121,10 +119,7 @@ select time_tz || '~' || - time_no_tz || '~' || - - - property_binary_data + time_no_tz ) as "_AIRBYTE_EXCHANGE_RATE_HASHID", tmp.* @@ -147,7 +142,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, "_AIRBYTE_AB_ID", "_AIRBYTE_EMITTED_AT", diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql index fe7ee3aa77ca..5eaf6186aaab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_c__lting_into_long_names_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_c__lting_into_long_names')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql index bbca6ea47254..c35233d432cb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/some_stream_that_was_empty_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('some_stream_that_was_empty')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('some_stream_that_was_empty')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('some_stream_that_was_empty')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('some_stream_that_was_empty')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays.sql index b5f10293fddf..875d02816862 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='arrays_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays_nested_array_parent.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays_nested_array_parent.sql index f3cfcf08e517..73f13e380ac2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays_nested_array_parent.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/arrays_nested_array_parent.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='arrays_nested_array_parent_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql index fb14fc0ec02f..ede71a891dc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_array.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='conflict_stream_array_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql index 518ddd4c8095..f203166febe1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='conflict_stream_name_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql index b83744e195fc..2c221c2940b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name___conflict_stream_name.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='conflict_stream_name___conflict_stream_name_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql index 02b9a967edd1..195d067ffe41 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_name_conflict_stream_name.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='conflict_stream_name_conflict_stream_name_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql index d737ff5acba8..31f263905b53 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/conflict_stream_scalar.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='conflict_stream_scalar_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql index 6451ca096b10..8b4cddcd4b17 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/non_nested_stream_wi__lting_into_long_names.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='non_nested_stream_wi__lting_into_long_names_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql index f1bacbbd7c45..7c113e7291b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='unnest_alias_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql index 2acf4b2bba57..ae4165f58160 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_childre__column___with__quotes.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='unnest_alias_childre__column___with__quotes_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql index a86cbfcd476c..9f98219880ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='unnest_alias_children_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql index 8fb89e2f4d14..14c766c3dd59 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_nested_streams/models/generated/airbyte_tables/test_normalization/unnest_alias_children_owner.sql @@ -1,22 +1,6 @@ {{ config( indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='unnest_alias_children_owner_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index e768371a824c..2773af0d8fa3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -21,7 +21,6 @@ select jsonb_extract_path_text(_airbyte_data, 'datetime_no_tz') as datetime_no_tz, jsonb_extract_path_text(_airbyte_data, 'time_tz') as time_tz, jsonb_extract_path_text(_airbyte_data, 'time_no_tz') as time_no_tz, - jsonb_extract_path_text(_airbyte_data, 'property_binary_data') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -66,7 +65,6 @@ select cast(nullif(time_no_tz, '') as time ) as time_no_tz, - cast(decode(property_binary_data, 'base64') as bytea) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -78,7 +76,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, + md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') as text)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -99,7 +97,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql index 0041cccaa66c..ca2b2520a258 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab1.sql @@ -20,7 +20,6 @@ select {{ json_extract_scalar('_airbyte_data', ['datetime_no_tz'], ['datetime_no_tz']) }} as datetime_no_tz, {{ json_extract_scalar('_airbyte_data', ['time_tz'], ['time_tz']) }} as time_tz, {{ json_extract_scalar('_airbyte_data', ['time_no_tz'], ['time_no_tz']) }} as time_no_tz, - {{ json_extract_scalar('_airbyte_data', ['property_binary_data'], ['property_binary_data']) }} as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql index e64542dc6d9b..0f457acbee98 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab2.sql @@ -20,7 +20,6 @@ select cast({{ empty_string_to_null('datetime_no_tz') }} as {{ type_timestamp_without_timezone() }}) as datetime_no_tz, cast({{ empty_string_to_null('time_tz') }} as {{ type_time_with_timezone() }}) as time_tz, cast({{ empty_string_to_null('time_no_tz') }} as {{ type_time_without_timezone() }}) as time_no_tz, - cast(decode(property_binary_data, 'base64') as {{ type_binary() }}) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql index fe3e0c53b765..789086fe147a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/exchange_rate_ab3.sql @@ -21,7 +21,6 @@ select 'datetime_no_tz', 'time_tz', 'time_no_tz', - 'property_binary_data', ]) }} as _airbyte_exchange_rate_hashid, tmp.* from {{ ref('exchange_rate_ab2') }} tmp diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/1_prefix_startwith_number_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/1_prefix_startwith_number_scd.sql index ba70ca94c1f1..01e0c49d1c7c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/1_prefix_startwith_number_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/1_prefix_startwith_number_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('1_prefix_startwith_number')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('1_prefix_startwith_number')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('1_prefix_startwith_number')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('1_prefix_startwith_number')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index c91e78cb81c2..5affe9825e3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index d0e7f0fd4a51..ef0cf7e1e95f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/multiple_column_names_conflicts_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/multiple_column_names_conflicts_scd.sql index ab9ddd969880..77d393c85689 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/multiple_column_names_conflicts_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/multiple_column_names_conflicts_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('multiple_column_names_conflicts')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('multiple_column_names_conflicts')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('multiple_column_names_conflicts')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('multiple_column_names_conflicts')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql index 074fd47f183f..ff471c6abaab 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/pos_dedup_cdcx_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('pos_dedup_cdcx')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('pos_dedup_cdcx')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('pos_dedup_cdcx')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('pos_dedup_cdcx')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 4ccea5b060a5..d8da713c6871 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql index 158c69afaa87..0a0b409c90b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('types_testing')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('types_testing')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index b1542a35dd2f..72e495678044 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -36,7 +20,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql index c91e78cb81c2..5affe9825e3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_cdc_excluded_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_cdc_excluded')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_cdc_excluded')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index f37cdb29d935..7e6225fb7cfc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql index 1d8d084ad443..96f720b3d265 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/renamed_dedup_cdc_excluded_scd.sql @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('renamed_dedup_cdc_excluded')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql index 3ac92c191bf4..40b5ffb3f87d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -2,22 +2,6 @@ indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], unique_key = '_airbyte_ab_id', schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index e768371a824c..2773af0d8fa3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -21,7 +21,6 @@ select jsonb_extract_path_text(_airbyte_data, 'datetime_no_tz') as datetime_no_tz, jsonb_extract_path_text(_airbyte_data, 'time_tz') as time_tz, jsonb_extract_path_text(_airbyte_data, 'time_no_tz') as time_no_tz, - jsonb_extract_path_text(_airbyte_data, 'property_binary_data') as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -66,7 +65,6 @@ select cast(nullif(time_no_tz, '') as time ) as time_no_tz, - cast(decode(property_binary_data, 'base64') as bytea) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at @@ -78,7 +76,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, + md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast("date" as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("HKD@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') as text)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -99,7 +97,6 @@ select datetime_no_tz, time_tz, time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, now() as _airbyte_normalized_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml index 8fdb7a50a697..767544968e0b 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml @@ -51,77 +51,77 @@ dispatch: vars: json_column: _airbyte_data models_to_source: - nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names: test_normalization_iprwf._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - some_stream_that_was_empty_ab1: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_ab2: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_stg: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_scd: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty: test_normalization_iprwf._airbyte_raw_some_stream_that_was_empty + nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names: test_normalization_xjvlg._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + some_stream_that_was_empty_ab1: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_ab2: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_stg: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_scd: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty: test_normalization_xjvlg._airbyte_raw_some_stream_that_was_empty simple_stream_with_namespace_resulting_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_scalar_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar: test_normalization_iprwf._airbyte_raw_conflict_stream_scalar - conflict_stream_array_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_array - conflict_stream_array_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_array - conflict_stream_array_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_array - conflict_stream_array: test_normalization_iprwf._airbyte_raw_conflict_stream_array - unnest_alias_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias: test_normalization_iprwf._airbyte_raw_unnest_alias - arrays_ab1: test_normalization_iprwf._airbyte_raw_arrays - arrays_ab2: test_normalization_iprwf._airbyte_raw_arrays - arrays_ab3: test_normalization_iprwf._airbyte_raw_arrays - arrays: test_normalization_iprwf._airbyte_raw_arrays - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name_conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name - unnest_alias_children_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children: test_normalization_iprwf._airbyte_raw_unnest_alias - arrays_nested_array_parent_ab1: test_normalization_iprwf._airbyte_raw_arrays - arrays_nested_array_parent_ab2: test_normalization_iprwf._airbyte_raw_arrays - arrays_nested_array_parent_ab3: test_normalization_iprwf._airbyte_raw_arrays - arrays_nested_array_parent: test_normalization_iprwf._airbyte_raw_arrays - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization_iprwf._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization_iprwf._airbyte_raw_conflict_stream_name - unnest_alias_children_owner_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab1: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab2: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab3: test_normalization_iprwf._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes: test_normalization_iprwf._airbyte_raw_unnest_alias + conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_scalar_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar: test_normalization_xjvlg._airbyte_raw_conflict_stream_scalar + conflict_stream_array_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_array + conflict_stream_array_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_array + conflict_stream_array_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_array + conflict_stream_array: test_normalization_xjvlg._airbyte_raw_conflict_stream_array + unnest_alias_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias: test_normalization_xjvlg._airbyte_raw_unnest_alias + arrays_ab1: test_normalization_xjvlg._airbyte_raw_arrays + arrays_ab2: test_normalization_xjvlg._airbyte_raw_arrays + arrays_ab3: test_normalization_xjvlg._airbyte_raw_arrays + arrays: test_normalization_xjvlg._airbyte_raw_arrays + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + conflict_stream_name_conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + unnest_alias_children_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children: test_normalization_xjvlg._airbyte_raw_unnest_alias + arrays_nested_array_parent_ab1: test_normalization_xjvlg._airbyte_raw_arrays + arrays_nested_array_parent_ab2: test_normalization_xjvlg._airbyte_raw_arrays + arrays_nested_array_parent_ab3: test_normalization_xjvlg._airbyte_raw_arrays + arrays_nested_array_parent: test_normalization_xjvlg._airbyte_raw_arrays + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization_xjvlg._airbyte_raw_conflict_stream_name + unnest_alias_children_owner_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab1: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab2: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab3: test_normalization_xjvlg._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes: test_normalization_xjvlg._airbyte_raw_unnest_alias diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql similarity index 85% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 272ce1475584..9b59d6d77c88 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" + "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) @@ -14,8 +14,8 @@ with input_data as ( select * - from "normalization_tests"._airbyte_test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_stg" - -- nested_stream_with_complex_columns_resulting_into_long_names from "normalization_tests".test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + from "integrationtests"._airbyte_test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_stg" + -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names ), scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index 9e9a1605e83c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "nested_stream_with_complex_columns_resulti__dbt_tmp185252546953" - ); - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp185252546953" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql new file mode 100644 index 000000000000..184fa2bf1104 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -0,0 +1,29 @@ + + + + create table + "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names" + + + compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) + + as ( + +-- Final base SQL model +-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" +select + _airbyte_unique_key, + id, + date, + "partition", + _airbyte_ab_id, + _airbyte_emitted_at, + getdate() as _airbyte_normalized_at, + _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid +from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_xjvlg._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index c14d680fc4a0..4e1c7b1f3942 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" + "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" compound sortkey(_airbyte_emitted_at) @@ -12,7 +12,7 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, "partition"."double_array_data" as double_array_data, @@ -20,7 +20,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias +from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and "partition" is not null @@ -62,7 +62,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_partition_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index 68ff8cc279d9..e19271e39a6f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" + "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" compound sortkey(_airbyte_emitted_at) @@ -12,13 +12,13 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" with joined as ( select table_alias._airbyte_partition_hashid as _airbyte_hashid, _airbyte_nested_data - from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data + from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data ) select _airbyte_partition_hashid, @@ -26,7 +26,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 @@ -67,7 +67,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql similarity index 78% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index fc777ebad858..7e38b76f87fe 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" + "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" compound sortkey(_airbyte_emitted_at) @@ -12,13 +12,13 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- depends_on: "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" with joined as ( select table_alias._airbyte_partition_hashid as _airbyte_hashid, _airbyte_nested_data - from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data + from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data ) select _airbyte_partition_hashid, @@ -26,7 +26,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 @@ -67,7 +67,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index e3a9ebcfe248..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "nested_stream_with_complex_columns_resulti__dbt_tmp185302582647" - ); - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") - ( - select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp185302582647" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 0c29b9812ff7..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") - ( - select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp185307922088" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index 63036c193614..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") - ( - select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp185315344939" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index f752bb2b1b74..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") - ( - select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp185315267996" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index a5df79b64a61..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,29 +0,0 @@ - - - - create table - "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names" - - - compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) - - as ( - --- Final base SQL model --- depends_on: "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" -select - _airbyte_unique_key, - id, - date, - "partition", - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" --- nested_stream_with_complex_columns_resulting_into_long_names from "normalization_tests".test_normalization_iprwf._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names -where 1 = 1 -and _airbyte_active_row = 1 - - ); - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql similarity index 83% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index 3fed7058e31b..ed49a5e91606 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_iprwf", + schema = "_airbyte_test_normalization_xjvlg", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +-- depends_on: {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, @@ -13,7 +13,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias +from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql similarity index 94% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 8e3c77dc93ae..19ab94bca151 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_iprwf", + schema = "_airbyte_test_normalization_xjvlg", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index a8ea2379d7b9..18a21b472981 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization_iprwf", + schema = "_airbyte_test_normalization_xjvlg", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql index 8ea9f6913c78..4cc3285a5f6e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization_iprwf", + schema = "_airbyte_test_normalization_xjvlg", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 521128185308..4876b27d7cc0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization_iprwf", + schema = "_airbyte_test_normalization_xjvlg", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql similarity index 92% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 5b2e75c95ffd..a629e4de4e5d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization_iprwf", + schema = "test_normalization_xjvlg", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('nested_stream_with_complex_columns_resulting_into_long_names')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization_iprwf.nested_stream_with_complex_columns_resulting_into_long_names_stg"], + ","drop view _airbyte_test_normalization_xjvlg.nested_stream_with_complex_columns_resulting_into_long_names_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }} - -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -95,7 +95,7 @@ input_data as ( input_data as ( select * from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }} - -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 56873588737d..f95f159eedc9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization_iprwf", + schema = "test_normalization_xjvlg", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -16,7 +16,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_iprwf', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_xjvlg', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 63a9656204a6..18a73cf63b7f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization_iprwf", + schema = "test_normalization_xjvlg", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql similarity index 94% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index 9d250f87e157..ad3d8a9a61b5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization_iprwf", + schema = "test_normalization_xjvlg", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index b82979e2eff0..2059cb60a01a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization_iprwf", + schema = "test_normalization_xjvlg", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml index 9effea5e1d45..56faa01c65dc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml @@ -1,6 +1,13 @@ version: 2 sources: -- name: test_normalization_iprwf +- name: test_normalization_namespace + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_simple_stream_with_namespace_resulting_into_long_names +- name: test_normalization_xjvlg quoting: database: true schema: false @@ -14,10 +21,3 @@ sources: - name: _airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - name: _airbyte_raw_some_stream_that_was_empty - name: _airbyte_raw_unnest_alias -- name: test_normalization_namespace - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_simple_stream_with_namespace_resulting_into_long_names diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql similarity index 50% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 5e536f58f6b8..45c63e057a5e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd" + delete from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "nested_stream_with_complex_columns_resulti__dbt_tmp" ); - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + insert into "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql deleted file mode 100644 index bfdfff49becd..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql similarity index 50% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 6a1aaba7abe1..e32bb140a099 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names" + delete from "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "nested_stream_with_complex_columns_resulti__dbt_tmp" ); - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + insert into "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") ( select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql new file mode 100644 index 000000000000..9944a91ca642 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -0,0 +1,9 @@ + + + + insert into "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") + ( + select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql new file mode 100644 index 000000000000..52b4bd4fc5f4 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -0,0 +1,9 @@ + + + + insert into "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") + ( + select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql new file mode 100644 index 000000000000..91aaa5e85cc0 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -0,0 +1,9 @@ + + + + insert into "integrationtests".test_normalization_xjvlg."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") + ( + select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" + from "nested_stream_with_complex_columns_resulti__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 16970f6dddcc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") - ( - select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index f755cf01912e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") - ( - select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 74f49febfe38..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_fxlbt/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_fxlbt."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") - ( - select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql deleted file mode 100644 index 62f298f7d815..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") - ( - select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql deleted file mode 100644 index 484c32aeb126..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") - ( - select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql deleted file mode 100644 index 15eec675847a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") - ( - select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql deleted file mode 100644 index 661feb40560e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization_iprwf/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ /dev/null @@ -1,9 +0,0 @@ - - - - insert into "normalization_tests".test_normalization_iprwf."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") - ( - select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" - from "nested_stream_with_complex_columns_resulti__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml index 5682537f147d..c645baf3c5fe 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml @@ -51,22 +51,22 @@ dispatch: vars: json_column: _airbyte_data models_to_source: - exchange_rate_ab1: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate: test_normalization_spffv._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded + exchange_rate_ab1: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate_ab2: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate_ab3: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate: test_normalization_bhhpj._airbyte_raw_exchange_rate + dedup_exchange_rate_ab1: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_ab2: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_stg: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_scd: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + renamed_dedup_cdc_excluded_ab1: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_ab2: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_stg: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_scd: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + dedup_cdc_excluded_ab1: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_ab2: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_stg: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_scd: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml index 2b0fdef69673..70d0b5b4fa3b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml @@ -51,42 +51,42 @@ dispatch: vars: json_column: _airbyte_data models_to_source: - exchange_rate_ab1: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization_spffv._airbyte_raw_exchange_rate - exchange_rate: test_normalization_spffv._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization_spffv._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization_spffv._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization_spffv._airbyte_raw_dedup_cdc_excluded - pos_dedup_cdcx_ab1: test_normalization_spffv._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_ab2: test_normalization_spffv._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_stg: test_normalization_spffv._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_scd: test_normalization_spffv._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx: test_normalization_spffv._airbyte_raw_pos_dedup_cdcx - 1_prefix_startwith_number_ab1: test_normalization_spffv._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_ab2: test_normalization_spffv._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_stg: test_normalization_spffv._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_scd: test_normalization_spffv._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number: test_normalization_spffv._airbyte_raw_1_prefix_startwith_number - multiple_column_names_conflicts_ab1: test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_ab2: test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_stg: test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_scd: test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts: test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts - types_testing_ab1: test_normalization_spffv._airbyte_raw_types_testing - types_testing_ab2: test_normalization_spffv._airbyte_raw_types_testing - types_testing_stg: test_normalization_spffv._airbyte_raw_types_testing - types_testing_scd: test_normalization_spffv._airbyte_raw_types_testing - types_testing: test_normalization_spffv._airbyte_raw_types_testing + exchange_rate_ab1: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate_ab2: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate_ab3: test_normalization_bhhpj._airbyte_raw_exchange_rate + exchange_rate: test_normalization_bhhpj._airbyte_raw_exchange_rate + dedup_exchange_rate_ab1: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_ab2: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_stg: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_scd: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate: test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate + renamed_dedup_cdc_excluded_ab1: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_ab2: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_stg: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_scd: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded: test_normalization_bhhpj._airbyte_raw_renamed_dedup_cdc_excluded + dedup_cdc_excluded_ab1: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_ab2: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_stg: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_scd: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded: test_normalization_bhhpj._airbyte_raw_dedup_cdc_excluded + pos_dedup_cdcx_ab1: test_normalization_bhhpj._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_ab2: test_normalization_bhhpj._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_stg: test_normalization_bhhpj._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_scd: test_normalization_bhhpj._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx: test_normalization_bhhpj._airbyte_raw_pos_dedup_cdcx + 1_prefix_startwith_number_ab1: test_normalization_bhhpj._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_ab2: test_normalization_bhhpj._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_stg: test_normalization_bhhpj._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_scd: test_normalization_bhhpj._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number: test_normalization_bhhpj._airbyte_raw_1_prefix_startwith_number + multiple_column_names_conflicts_ab1: test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_ab2: test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_stg: test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_scd: test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts: test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization_bhhpj._airbyte_raw_types_testing + types_testing_ab2: test_normalization_bhhpj._airbyte_raw_types_testing + types_testing_stg: test_normalization_bhhpj._airbyte_raw_types_testing + types_testing_scd: test_normalization_bhhpj._airbyte_raw_types_testing + types_testing: test_normalization_bhhpj._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 01a95283d6fa..3c1032d3297f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" + "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) @@ -14,8 +14,8 @@ with input_data as ( select * - from "normalization_tests"._airbyte_test_normalization_spffv."dedup_exchange_rate_stg" - -- dedup_exchange_rate from "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate + from "integrationtests"._airbyte_test_normalization_bhhpj."dedup_exchange_rate_stg" + -- dedup_exchange_rate from "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate ), scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql deleted file mode 100644 index b7fc853243ac..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "dedup_exchange_rate_scd__dbt_tmp185609343121" - ); - - - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate_scd__dbt_tmp185609343121" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 58% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 4ce40ca0bb16..b6903fe4ceb0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -2,7 +2,7 @@ create table - "normalization_tests".test_normalization_spffv."dedup_exchange_rate" + "integrationtests".test_normalization_bhhpj."dedup_exchange_rate" compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) @@ -10,7 +10,7 @@ as ( -- Final base SQL model --- depends_on: "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" +-- depends_on: "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" select _airbyte_unique_key, id, @@ -25,8 +25,8 @@ select _airbyte_emitted_at, getdate() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid -from "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" --- dedup_exchange_rate from "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate +from "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate where 1 = 1 and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql deleted file mode 100644 index 17918c5d32d8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "dedup_exchange_rate__dbt_tmp185631855953" - ); - - - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate__dbt_tmp185631855953" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql similarity index 71% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index dfa229adebae..e2bd3830cb42 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "normalization_tests".test_normalization_vorny."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_bhhpj."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,21 +11,21 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."new_column" != '' then _airbyte_data."new_column" end as new_column, case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", + case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, case when _airbyte_data."column`_'with""_quotes" != '' then _airbyte_data."column`_'with""_quotes" end as "column`_'with""_quotes", _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -34,21 +34,19 @@ where 1 = 1 -- depends_on: __dbt__cte__exchange_rate_ab1 select cast(id as - float + bigint ) as id, cast(currency as text) as currency, - cast(new_column as - float -) as new_column, cast(nullif(date::varchar, '') as date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float ) as "hkd@spéçiäl & characters", + cast(hkd_special___characters as text) as hkd_special___characters, cast(nzd as float ) as nzd, @@ -67,7 +65,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(new_column as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') as text)) as _airbyte_exchange_rate_hashid, + md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') as text)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -77,10 +75,10 @@ where 1 = 1 select id, currency, - new_column, date, timestamp_col, "hkd@spéçiäl & characters", + hkd_special___characters, nzd, usd, "column`_'with""_quotes", @@ -89,6 +87,6 @@ select getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql deleted file mode 100644 index f0cbba5b655d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql +++ /dev/null @@ -1,115 +0,0 @@ - - - create table - "normalization_tests".test_normalization_spffv."exchange_rate__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - - as ( - -with __dbt__cte__exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - case when _airbyte_data."column`_'with""_quotes" != '' then _airbyte_data."column`_'with""_quotes" end as "column`_'with""_quotes", - case when _airbyte_data."datetime_tz" != '' then _airbyte_data."datetime_tz" end as datetime_tz, - case when _airbyte_data."datetime_no_tz" != '' then _airbyte_data."datetime_no_tz" end as datetime_no_tz, - case when _airbyte_data."time_tz" != '' then _airbyte_data."time_tz" end as time_tz, - case when _airbyte_data."time_no_tz" != '' then _airbyte_data."time_no_tz" end as time_no_tz, - case when _airbyte_data."property_binary_data" != '' then _airbyte_data."property_binary_data" end as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__exchange_rate_ab1 -select - cast(id as - bigint -) as id, - cast(currency as text) as currency, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as text) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - cast("column`_'with""_quotes" as text) as "column`_'with""_quotes", - cast(nullif(datetime_tz::varchar, '') as - TIMESTAMPTZ -) as datetime_tz, - cast(nullif(datetime_no_tz::varchar, '') as - TIMESTAMP -) as datetime_no_tz, - cast(nullif(time_tz::varchar, '') as - TIMETZ -) as time_tz, - cast(nullif(time_no_tz::varchar, '') as - TIME -) as time_no_tz, - cast(property_binary_data as text) as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__exchange_rate_ab1 --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, - tmp.* -from __dbt__cte__exchange_rate_ab2 tmp --- exchange_rate -where 1 = 1 -)-- Final base SQL model --- depends_on: __dbt__cte__exchange_rate_ab3 -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "column`_'with""_quotes", - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql deleted file mode 100644 index 8e60b4e38a6a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql +++ /dev/null @@ -1,115 +0,0 @@ - - - create table - "normalization_tests".test_normalization_vorny."exchange_rate__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - - as ( - -with __dbt__cte__exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - case when _airbyte_data."column`_'with""_quotes" != '' then _airbyte_data."column`_'with""_quotes" end as "column`_'with""_quotes", - case when _airbyte_data."datetime_tz" != '' then _airbyte_data."datetime_tz" end as datetime_tz, - case when _airbyte_data."datetime_no_tz" != '' then _airbyte_data."datetime_no_tz" end as datetime_no_tz, - case when _airbyte_data."time_tz" != '' then _airbyte_data."time_tz" end as time_tz, - case when _airbyte_data."time_no_tz" != '' then _airbyte_data."time_no_tz" end as time_no_tz, - case when _airbyte_data."property_binary_data" != '' then _airbyte_data."property_binary_data" end as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__exchange_rate_ab1 -select - cast(id as - bigint -) as id, - cast(currency as text) as currency, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as text) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - cast("column`_'with""_quotes" as text) as "column`_'with""_quotes", - cast(nullif(datetime_tz::varchar, '') as - TIMESTAMPTZ -) as datetime_tz, - cast(nullif(datetime_no_tz::varchar, '') as - TIMESTAMP -) as datetime_no_tz, - cast(nullif(time_tz::varchar, '') as - TIMETZ -) as time_tz, - cast(nullif(time_no_tz::varchar, '') as - TIME -) as time_no_tz, - cast(property_binary_data as text) as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__exchange_rate_ab1 --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, - tmp.* -from __dbt__cte__exchange_rate_ab2 tmp --- exchange_rate -where 1 = 1 -)-- Final base SQL model --- depends_on: __dbt__cte__exchange_rate_ab3 -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "column`_'with""_quotes", - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 460ee061096a..903a3141f625 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "normalization_tests"._airbyte_test_normalization_spffv."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_bhhpj."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 @@ -35,7 +35,7 @@ select date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/multiple_column_names_conflicts_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql index cdcb4ed52ddb..b496abf0c5ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/multiple_column_names_conflicts_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql @@ -1,11 +1,11 @@ - create view "normalization_tests"._airbyte_test_normalization_vorny."multiple_column_names_conflicts_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_bhhpj."multiple_column_names_conflicts_stg__dbt_tmp" as ( with __dbt__cte__multiple_column_names_conflicts_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_multiple_column_names_conflicts +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."User Id" != '' then _airbyte_data."User Id" end as "user id", @@ -17,7 +17,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_multiple_column_names_conflicts as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_multiple_column_names_conflicts as table_alias -- multiple_column_names_conflicts where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/multiple_column_names_conflicts_stg.sql deleted file mode 100644 index 658659ec2b8e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_spffv/multiple_column_names_conflicts_stg.sql +++ /dev/null @@ -1,62 +0,0 @@ - - - create view "normalization_tests"._airbyte_test_normalization_spffv."multiple_column_names_conflicts_stg__dbt_tmp" as ( - -with __dbt__cte__multiple_column_names_conflicts_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."User Id" != '' then _airbyte_data."User Id" end as "user id", - case when _airbyte_data."user_id" != '' then _airbyte_data."user_id" end as user_id, - case when _airbyte_data."User id" != '' then _airbyte_data."User id" end as "user id_1", - case when _airbyte_data."user id" != '' then _airbyte_data."user id" end as "user id_2", - case when _airbyte_data."User@Id" != '' then _airbyte_data."User@Id" end as "user@id", - case when _airbyte_data."UserId" != '' then _airbyte_data."UserId" end as userid, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_multiple_column_names_conflicts as table_alias --- multiple_column_names_conflicts -where 1 = 1 - -), __dbt__cte__multiple_column_names_conflicts_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__multiple_column_names_conflicts_ab1 -select - cast(id as - bigint -) as id, - cast("user id" as text) as "user id", - cast(user_id as - float -) as user_id, - cast("user id_1" as - float -) as "user id_1", - cast("user id_2" as - float -) as "user id_2", - cast("user@id" as text) as "user@id", - cast(userid as - float -) as userid, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__multiple_column_names_conflicts_ab1 --- multiple_column_names_conflicts -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__multiple_column_names_conflicts_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast("user id" as text), '') || '-' || coalesce(cast(user_id as text), '') || '-' || coalesce(cast("user id_1" as text), '') || '-' || coalesce(cast("user id_2" as text), '') || '-' || coalesce(cast("user@id" as text), '') || '-' || coalesce(cast(userid as text), '') as text)) as _airbyte_multiple_column_names_conflicts_hashid, - tmp.* -from __dbt__cte__multiple_column_names_conflicts_ab2 tmp --- multiple_column_names_conflicts -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql deleted file mode 100644 index 7cb258900a1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - create view "normalization_tests"._airbyte_test_normalization_vorny."dedup_exchange_rate_stg__dbt_tmp" as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__dedup_exchange_rate_ab1 -select - cast(id as - bigint -) as id, - cast(currency as text) as currency, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as text) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__dedup_exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') as text)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 0892b57e079b..b8200f8bf679 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} +-- depends_on: {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 6b5d4f79462e..420c7c986975 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql similarity index 92% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index febb267c3980..b716e29bdf6e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization_spffv", + schema = "test_normalization_bhhpj", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization_spffv.dedup_exchange_rate_stg"], + ","drop view _airbyte_test_normalization_bhhpj.dedup_exchange_rate_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('dedup_exchange_rate_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -97,7 +97,7 @@ input_data as ( input_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index f3ceea61d3af..8f8fd8c8e9bc 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization_spffv", + schema = "test_normalization_bhhpj", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,7 +21,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} +-- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..a66a0b168c2e --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "test_normalization_bhhpj", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('exchange_rate_ab3') }} +select + id, + currency, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + hkd_special___characters, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql deleted file mode 100644 index 1a1460e51a6f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql +++ /dev/null @@ -1,47 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = '_airbyte_ab_id', - schema = "test_normalization_spffv", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], - tags = [ "top-level" ] -) }} --- Final base SQL model --- depends_on: {{ ref('exchange_rate_ab3') }} -select - id, - currency, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('column`_\'with""_quotes') }}, - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_exchange_rate') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 93% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index b20dfdcfd999..db45cc80a67a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml index 3b4123fceff3..6aa768851a80 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml @@ -1,6 +1,6 @@ version: 2 sources: -- name: test_normalization_spffv +- name: test_normalization_bhhpj quoting: database: true schema: false diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index ffff62aa4e59..cfb1d029d88f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} +-- depends_on: {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql similarity index 95% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 7a309e0955fa..2a9275c69a1e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization_spffv/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql similarity index 92% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 05de45b0a0a7..9f8c382ff834 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization_spffv", + schema = "test_normalization_bhhpj", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -33,12 +33,12 @@ from ( select distinct _airbyte_unique_key as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} ) recent_records left join ( select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('dedup_exchange_rate')) }} group by _airbyte_unique_key ) active_counts on recent_records.unique_key = active_counts.unique_key @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization_spffv.dedup_exchange_rate_stg"], + ","drop view _airbyte_test_normalization_bhhpj.dedup_exchange_rate_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('dedup_exchange_rate_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -97,7 +97,7 @@ input_data as ( input_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 87% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 14a199ed47a8..c5fed3b30237 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization_spffv", + schema = "test_normalization_bhhpj", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,7 +21,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_dedup_exchange_rate') }} +-- dedup_exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql new file mode 100644 index 000000000000..9a7a498cc375 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -0,0 +1,26 @@ +{{ config( + sort = "_airbyte_emitted_at", + unique_key = '_airbyte_ab_id', + schema = "test_normalization_bhhpj", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('exchange_rate_ab3') }} +select + id, + currency, + new_column, + date, + timestamp_col, + {{ adapter.quote('hkd@spéçiäl & characters') }}, + nzd, + usd, + {{ adapter.quote('column`_\'with""_quotes') }}, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_exchange_rate_hashid +from {{ ref('exchange_rate_ab3') }} +-- exchange_rate from {{ source('test_normalization_bhhpj', '_airbyte_raw_exchange_rate') }} +where 1 = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql deleted file mode 100644 index 6d587ef82dcc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization_spffv/exchange_rate.sql +++ /dev/null @@ -1,42 +0,0 @@ -{{ config( - sort = "_airbyte_emitted_at", - unique_key = '_airbyte_ab_id', - schema = "test_normalization_spffv", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], - tags = [ "top-level" ] -) }} --- Final base SQL model --- depends_on: {{ ref('exchange_rate_ab3') }} -select - id, - currency, - new_column, - date, - timestamp_col, - {{ adapter.quote('hkd@spéçiäl & characters') }}, - nzd, - usd, - {{ adapter.quote('column`_\'with""_quotes') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization_spffv', '_airbyte_raw_exchange_rate') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 93% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index a2070c639bc4..9d10a9ea9490 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization_spffv", + schema = "_airbyte_test_normalization_bhhpj", tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml index 3d8790bc8163..4daf898b3002 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml @@ -1,6 +1,6 @@ version: 2 sources: -- name: test_normalization_spffv +- name: test_normalization_bhhpj quoting: database: true schema: false diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index ea7873846f77..de775a2e5c16 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" + delete from "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "dedup_exchange_rate_scd__dbt_tmp" ); - insert into "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate_scd__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql deleted file mode 100644 index ae44ccfc5883..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "dedup_exchange_rate_scd__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate_scd__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 24a1407f649b..372889fb42bd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate" + delete from "integrationtests".test_normalization_bhhpj."dedup_exchange_rate" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "dedup_exchange_rate__dbt_tmp" ); - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_bhhpj."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql deleted file mode 100644 index eb09558982a0..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_spffv."dedup_exchange_rate" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "dedup_exchange_rate__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_spffv."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql similarity index 66% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index f0cbba5b655d..e2bd3830cb42 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "normalization_tests".test_normalization_spffv."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_bhhpj."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,7 +11,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -22,15 +22,10 @@ select case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, case when _airbyte_data."column`_'with""_quotes" != '' then _airbyte_data."column`_'with""_quotes" end as "column`_'with""_quotes", - case when _airbyte_data."datetime_tz" != '' then _airbyte_data."datetime_tz" end as datetime_tz, - case when _airbyte_data."datetime_no_tz" != '' then _airbyte_data."datetime_no_tz" end as datetime_no_tz, - case when _airbyte_data."time_tz" != '' then _airbyte_data."time_tz" end as time_tz, - case when _airbyte_data."time_no_tz" != '' then _airbyte_data."time_no_tz" end as time_no_tz, - case when _airbyte_data."property_binary_data" != '' then _airbyte_data."property_binary_data" end as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -46,7 +41,7 @@ select date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float @@ -59,19 +54,6 @@ select float ) as usd, cast("column`_'with""_quotes" as text) as "column`_'with""_quotes", - cast(nullif(datetime_tz::varchar, '') as - TIMESTAMPTZ -) as datetime_tz, - cast(nullif(datetime_no_tz::varchar, '') as - TIMESTAMP -) as datetime_no_tz, - cast(nullif(time_tz::varchar, '') as - TIMETZ -) as time_tz, - cast(nullif(time_no_tz::varchar, '') as - TIME -) as time_no_tz, - cast(property_binary_data as text) as property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at @@ -83,7 +65,7 @@ where 1 = 1 -- SQL model to build a hash column based on the values of this record -- depends_on: __dbt__cte__exchange_rate_ab2 select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, + md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') as text)) as _airbyte_exchange_rate_hashid, tmp.* from __dbt__cte__exchange_rate_ab2 tmp -- exchange_rate @@ -100,16 +82,11 @@ select nzd, usd, "column`_'with""_quotes", - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql deleted file mode 100644 index 8e60b4e38a6a..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization_vorny/exchange_rate.sql +++ /dev/null @@ -1,115 +0,0 @@ - - - create table - "normalization_tests".test_normalization_vorny."exchange_rate__dbt_tmp" - - - compound sortkey(_airbyte_emitted_at) - - as ( - -with __dbt__cte__exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - case when _airbyte_data."column`_'with""_quotes" != '' then _airbyte_data."column`_'with""_quotes" end as "column`_'with""_quotes", - case when _airbyte_data."datetime_tz" != '' then _airbyte_data."datetime_tz" end as datetime_tz, - case when _airbyte_data."datetime_no_tz" != '' then _airbyte_data."datetime_no_tz" end as datetime_no_tz, - case when _airbyte_data."time_tz" != '' then _airbyte_data."time_tz" end as time_tz, - case when _airbyte_data."time_no_tz" != '' then _airbyte_data."time_no_tz" end as time_no_tz, - case when _airbyte_data."property_binary_data" != '' then _airbyte_data."property_binary_data" end as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate as table_alias --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__exchange_rate_ab1 -select - cast(id as - bigint -) as id, - cast(currency as text) as currency, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as text) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - cast("column`_'with""_quotes" as text) as "column`_'with""_quotes", - cast(nullif(datetime_tz::varchar, '') as - TIMESTAMPTZ -) as datetime_tz, - cast(nullif(datetime_no_tz::varchar, '') as - TIMESTAMP -) as datetime_no_tz, - cast(nullif(time_tz::varchar, '') as - TIMETZ -) as time_tz, - cast(nullif(time_no_tz::varchar, '') as - TIME -) as time_no_tz, - cast(property_binary_data as text) as property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__exchange_rate_ab1 --- exchange_rate -where 1 = 1 -), __dbt__cte__exchange_rate_ab3 as ( - --- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') || '-' || coalesce(cast("column`_'with""_quotes" as text), '') || '-' || coalesce(cast(datetime_tz as text), '') || '-' || coalesce(cast(datetime_no_tz as text), '') || '-' || coalesce(cast(time_tz as text), '') || '-' || coalesce(cast(time_no_tz as text), '') || '-' || coalesce(cast(property_binary_data as text), '') as text)) as _airbyte_exchange_rate_hashid, - tmp.* -from __dbt__cte__exchange_rate_ab2 tmp --- exchange_rate -where 1 = 1 -)-- Final base SQL model --- depends_on: __dbt__cte__exchange_rate_ab3 -select - id, - currency, - date, - timestamp_col, - "hkd@spéçiäl & characters", - hkd_special___characters, - nzd, - usd, - "column`_'with""_quotes", - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_vorny._airbyte_raw_exchange_rate -where 1 = 1 - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 460ee061096a..903a3141f625 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "normalization_tests"._airbyte_test_normalization_spffv."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_bhhpj."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 @@ -35,7 +35,7 @@ select date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql deleted file mode 100644 index 7cb258900a1f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,66 +0,0 @@ - - - create view "normalization_tests"._airbyte_test_normalization_vorny."dedup_exchange_rate_stg__dbt_tmp" as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."HKD_special___characters" != '' then _airbyte_data."HKD_special___characters" end as hkd_special___characters, - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__dedup_exchange_rate_ab1 -select - cast(id as - bigint -) as id, - cast(currency as text) as currency, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(hkd_special___characters as text) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__dedup_exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(hkd_special___characters as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') as text)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index e37635979538..a193db25eb23 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_vorny/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" + delete from "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "dedup_exchange_rate_scd__dbt_tmp" ); - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_bhhpj."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate_scd__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql deleted file mode 100644 index d9f559ef203d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization_spffv/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" - where (_airbyte_unique_key_scd) in ( - select (_airbyte_unique_key_scd) - from "dedup_exchange_rate_scd__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_spffv."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate_scd__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql similarity index 52% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 2d11459f8b4b..6afa610cc721 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_spffv/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,13 @@ - delete from "normalization_tests".test_normalization_spffv."dedup_exchange_rate" + delete from "integrationtests".test_normalization_bhhpj."dedup_exchange_rate" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "dedup_exchange_rate__dbt_tmp" ); - insert into "normalization_tests".test_normalization_spffv."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_bhhpj."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql deleted file mode 100644 index 1f35d7840bee..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization_vorny/dedup_exchange_rate.sql +++ /dev/null @@ -1,15 +0,0 @@ - - - delete from "normalization_tests".test_normalization_vorny."dedup_exchange_rate" - where (_airbyte_unique_key) in ( - select (_airbyte_unique_key) - from "dedup_exchange_rate__dbt_tmp" - ); - - - insert into "normalization_tests".test_normalization_vorny."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") - ( - select "_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" - from "dedup_exchange_rate__dbt_tmp" - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql similarity index 89% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql index c8e0c37d6d6e..031baa2a7efb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization_spffv/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "normalization_tests".test_normalization_spffv."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_bhhpj."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,7 +11,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -25,7 +25,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -44,7 +44,7 @@ select date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float @@ -89,6 +89,6 @@ select getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "normalization_tests".test_normalization_spffv._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_bhhpj._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql similarity index 88% rename from airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql rename to airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 509c38a1c750..8c9d36dd07d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_spffv/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "normalization_tests"._airbyte_test_normalization_spffv."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_bhhpj."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_spffv._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_bhhpj._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 @@ -38,7 +38,7 @@ select date ) as date, cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ + timestamp with time zone ) as timestamp_col, cast("hkd@spéçiäl & characters" as float diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql deleted file mode 100644 index 545ff13328cf..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization_vorny/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,68 +0,0 @@ - - - create view "normalization_tests"._airbyte_test_normalization_vorny."dedup_exchange_rate_stg__dbt_tmp" as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate -select - case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, - case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, - case when _airbyte_data."new_column" != '' then _airbyte_data."new_column" end as new_column, - case when _airbyte_data."date" != '' then _airbyte_data."date" end as date, - case when _airbyte_data."timestamp_col" != '' then _airbyte_data."timestamp_col" end as timestamp_col, - case when _airbyte_data."HKD@spéçiäl & characters" != '' then _airbyte_data."HKD@spéçiäl & characters" end as "hkd@spéçiäl & characters", - case when _airbyte_data."NZD" != '' then _airbyte_data."NZD" end as nzd, - case when _airbyte_data."USD" != '' then _airbyte_data."USD" end as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from "normalization_tests".test_normalization_vorny._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__dedup_exchange_rate_ab1 -select - cast(id as - float -) as id, - cast(currency as text) as currency, - cast(new_column as - float -) as new_column, - cast(nullif(date::varchar, '') as - date -) as date, - cast(nullif(timestamp_col::varchar, '') as - TIMESTAMPTZ -) as timestamp_col, - cast("hkd@spéçiäl & characters" as - float -) as "hkd@spéçiäl & characters", - cast(nzd as - float -) as nzd, - cast(usd as - bigint -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - getdate() as _airbyte_normalized_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__dedup_exchange_rate_ab2 -select - md5(cast(coalesce(cast(id as text), '') || '-' || coalesce(cast(currency as text), '') || '-' || coalesce(cast(new_column as text), '') || '-' || coalesce(cast(date as text), '') || '-' || coalesce(cast(timestamp_col as text), '') || '-' || coalesce(cast("hkd@spéçiäl & characters" as text), '') || '-' || coalesce(cast(nzd as text), '') || '-' || coalesce(cast(usd as text), '') as text)) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -where 1 = 1 - - ) ; diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql index a8c3a29ba971..7b46e390d057 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_SCD.sql @@ -33,12 +33,12 @@ from ( select distinct _AIRBYTE_UNIQUE_KEY as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', adapter.quote(this.schema) + '.' + adapter.quote('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES')) }} + where 1=1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', this.schema + '.' + adapter.quote('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES')) }} ) recent_records left join ( select _AIRBYTE_UNIQUE_KEY as unique_key, count(_AIRBYTE_UNIQUE_KEY) as active_count from {{ this }} - where _AIRBYTE_ACTIVE_ROW = 1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', adapter.quote(this.schema) + '.' + adapter.quote('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES')) }} + where _AIRBYTE_ACTIVE_ROW = 1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', this.schema + '.' + adapter.quote('NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES')) }} group by _AIRBYTE_UNIQUE_KEY ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index 38666ffd4727..e35addfdeb76 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/first_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -21,7 +21,6 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"datetime_no_tz"')) as DATETIME_NO_TZ, to_varchar(get_path(parse_json(_airbyte_data), '"time_tz"')) as TIME_TZ, to_varchar(get_path(parse_json(_airbyte_data), '"time_no_tz"')) as TIME_NO_TZ, - to_varchar(get_path(parse_json(_airbyte_data), '"property_binary_data"')) as PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT @@ -88,7 +87,6 @@ select cast(nullif(TIME_NO_TZ, '') as time ) as TIME_NO_TZ, - cast(BASE64_DECODE_BINARY(PROPERTY_BINARY_DATA) as VARBINARY) as PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT @@ -126,8 +124,6 @@ select varchar ), '') || '-' || coalesce(cast(TIME_NO_TZ as varchar -), '') || '-' || coalesce(cast(PROPERTY_BINARY_DATA as - varchar ), '') as varchar )) as _AIRBYTE_EXCHANGE_RATE_HASHID, @@ -151,7 +147,6 @@ select DATETIME_NO_TZ, TIME_TZ, TIME_NO_TZ, - PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql index 85663304b440..13f493601511 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_incremental/scd/TEST_NORMALIZATION/DEDUP_EXCHANGE_RATE_SCD.sql @@ -33,12 +33,12 @@ from ( select distinct _AIRBYTE_UNIQUE_KEY as unique_key from {{ this }} - where 1=1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', adapter.quote(this.schema) + '.' + adapter.quote('DEDUP_EXCHANGE_RATE')) }} + where 1=1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', this.schema + '.' + adapter.quote('DEDUP_EXCHANGE_RATE')) }} ) recent_records left join ( select _AIRBYTE_UNIQUE_KEY as unique_key, count(_AIRBYTE_UNIQUE_KEY) as active_count from {{ this }} - where _AIRBYTE_ACTIVE_ROW = 1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', adapter.quote(this.schema) + '.' + adapter.quote('DEDUP_EXCHANGE_RATE')) }} + where _AIRBYTE_ACTIVE_ROW = 1 {{ incremental_clause('_AIRBYTE_NORMALIZED_AT', this.schema + '.' + adapter.quote('DEDUP_EXCHANGE_RATE')) }} group by _AIRBYTE_UNIQUE_KEY ) active_counts on recent_records.unique_key = active_counts.unique_key diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index 60148a141461..6b42adb3962d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -2,22 +2,6 @@ cluster_by = ["_AIRBYTE_EMITTED_AT"], unique_key = '_AIRBYTE_AB_ID', schema = "TEST_NORMALIZATION", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='EXCHANGE_RATE_SCD' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], tags = [ "top-level" ] ) }} -- Final base SQL model @@ -36,7 +20,6 @@ select DATETIME_NO_TZ, TIME_TZ, TIME_NO_TZ, - PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, {{ current_timestamp() }} as _AIRBYTE_NORMALIZED_AT, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql index 38666ffd4727..e35addfdeb76 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/second_output/airbyte_tables/TEST_NORMALIZATION/EXCHANGE_RATE.sql @@ -21,7 +21,6 @@ select to_varchar(get_path(parse_json(_airbyte_data), '"datetime_no_tz"')) as DATETIME_NO_TZ, to_varchar(get_path(parse_json(_airbyte_data), '"time_tz"')) as TIME_TZ, to_varchar(get_path(parse_json(_airbyte_data), '"time_no_tz"')) as TIME_NO_TZ, - to_varchar(get_path(parse_json(_airbyte_data), '"property_binary_data"')) as PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT @@ -88,7 +87,6 @@ select cast(nullif(TIME_NO_TZ, '') as time ) as TIME_NO_TZ, - cast(BASE64_DECODE_BINARY(PROPERTY_BINARY_DATA) as VARBINARY) as PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT @@ -126,8 +124,6 @@ select varchar ), '') || '-' || coalesce(cast(TIME_NO_TZ as varchar -), '') || '-' || coalesce(cast(PROPERTY_BINARY_DATA as - varchar ), '') as varchar )) as _AIRBYTE_EXCHANGE_RATE_HASHID, @@ -151,7 +147,6 @@ select DATETIME_NO_TZ, TIME_TZ, TIME_NO_TZ, - PROPERTY_BINARY_DATA, _AIRBYTE_AB_ID, _AIRBYTE_EMITTED_AT, convert_timezone('UTC', current_timestamp()) as _AIRBYTE_NORMALIZED_AT, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/dbt_project.yml deleted file mode 100755 index b43b3dad276e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/dbt_project.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: airbyte_utils -version: '1.0' -config-version: 2 -profile: normalize -model-paths: -- models -docs-paths: -- docs -analysis-paths: -- analysis -test-paths: -- tests -seed-paths: -- data -macro-paths: -- macros -target-path: ../build -log-path: ../logs -packages-install-path: /dbt -clean-targets: -- build -- dbt_modules -quoting: - database: true - schema: false - identifier: true -models: - airbyte_utils: - +materialized: table - generated: - airbyte_ctes: - +tags: airbyte_internal_cte - +materialized: ephemeral - airbyte_incremental: - +tags: incremental_tables - +materialized: incremental - airbyte_tables: - +tags: normalized_tables - +materialized: table - airbyte_views: - +tags: airbyte_internal_views - +materialized: view -vars: - dbt_utils_dispatch_list: - - airbyte_utils - json_column: _airbyte_data - models_to_source: - nested_stream_with_co_1g_into_long_names_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_1g_into_long_names_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_1g_into_long_names_stg: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_1g_into_long_names_scd: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co__lting_into_long_names: test_normalization._airbyte_raw_nested_s__lting_into_long_names - non_nested_stream_wit_1g_into_long_names_ab1: test_normalization._airbyte_raw_non_nest__lting_into_long_names - non_nested_stream_wit_1g_into_long_names_ab2: test_normalization._airbyte_raw_non_nest__lting_into_long_names - non_nested_stream_wit_1g_into_long_names_ab3: test_normalization._airbyte_raw_non_nest__lting_into_long_names - non_nested_stream_wit__lting_into_long_names: test_normalization._airbyte_raw_non_nest__lting_into_long_names - some_stream_that_was_empty_ab1: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_ab2: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_stg: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_scd: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty: test_normalization._airbyte_raw_some_stream_that_was_empty - simple_stream_with_na_1g_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names - simple_stream_with_na_1g_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names - simple_stream_with_na_1g_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names - simple_stream_with_na__lting_into_long_names: test_normalization_namespace._airbyte_raw_simple_s__lting_into_long_names - conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_scalar_ab1: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab2: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab3: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_array_ab1: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab2: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab3: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array: test_normalization._airbyte_raw_conflict_stream_array - unnest_alias_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias: test_normalization._airbyte_raw_unnest_alias - arrays_ab1: test_normalization._airbyte_raw_arrays - arrays_ab2: test_normalization._airbyte_raw_arrays - arrays_ab3: test_normalization._airbyte_raw_arrays - arrays: test_normalization._airbyte_raw_arrays - nested_stream_with_co_2g_names_partition_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_2g_names_partition_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_2g_names_partition_ab3: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co___long_names_partition: test_normalization._airbyte_raw_nested_s__lting_into_long_names - conflict_stream_name__2flict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name__2flict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name__2flict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children: test_normalization._airbyte_raw_unnest_alias - arrays_nested_array_parent_ab1: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent_ab2: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent_ab3: test_normalization._airbyte_raw_arrays - arrays_nested_array_parent: test_normalization._airbyte_raw_arrays - nested_stream_with_co_3double_array_data_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_3double_array_data_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_3double_array_data_ab3: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co__ion_double_array_data: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_3es_partition_data_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_3es_partition_data_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co_3es_partition_data_ab3: test_normalization._airbyte_raw_nested_s__lting_into_long_names - nested_stream_with_co___names_partition_data: test_normalization._airbyte_raw_nested_s__lting_into_long_names - conflict_stream_name__3flict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name__3flict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name__3flict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name____conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_owner_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_4mn___with__quotes_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_4mn___with__quotes_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_4mn___with__quotes_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children__column___with__quotes: test_normalization._airbyte_raw_unnest_alias diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql deleted file mode 100644 index d638e7a898ff..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - {{ json_extract('table_alias', '_airbyte_data', ['partition'], ['partition']) }} as {{ adapter.quote('partition') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} as table_alias --- nested_stream_with_co__lting_into_long_names -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql deleted file mode 100644 index b688e0746faa..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_1g_into_long_names_ab2.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: {{ ref('nested_stream_with_co_1g_into_long_names_ab1') }} -select - cast(id as {{ dbt_utils.type_string() }}) as id, - cast({{ adapter.quote('date') }} as {{ dbt_utils.type_string() }}) as {{ adapter.quote('date') }}, - cast({{ adapter.quote('partition') }} as {{ type_json() }}) as {{ adapter.quote('partition') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_1g_into_long_names_ab1') }} --- nested_stream_with_co__lting_into_long_names -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql deleted file mode 100644 index 427a929211b2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_2g_names_partition_ab1.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} -select - _airbyte_nested_strea__nto_long_names_hashid, - {{ json_extract_array(adapter.quote('partition'), ['double_array_data'], ['double_array_data']) }} as double_array_data, - {{ json_extract_array(adapter.quote('partition'), ['DATA'], ['DATA']) }} as {{ adapter.quote('DATA') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} as table_alias --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition -where 1 = 1 -and {{ adapter.quote('partition') }} is not null -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql deleted file mode 100644 index a8ca4bbb7d40..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3double_array_data_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ ref('nested_stream_with_co___long_names_partition') }} -{{ unnest_cte(ref('nested_stream_with_co___long_names_partition'), 'partition', 'double_array_data') }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value('double_array_data'), ['id'], ['id']) }} as id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data -{{ cross_join_unnest('partition', 'double_array_data') }} -where 1 = 1 -and double_array_data is not null -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql deleted file mode 100644 index cdf1151ee10d..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_co_3es_partition_data_ab1.sql +++ /dev/null @@ -1,20 +0,0 @@ -{{ config( - schema = "_airbyte_test_normalization", - tags = [ "nested-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ ref('nested_stream_with_co___long_names_partition') }} -{{ unnest_cte(ref('nested_stream_with_co___long_names_partition'), 'partition', adapter.quote('DATA')) }} -select - _airbyte_partition_hashid, - {{ json_extract_scalar(unnested_column_value(adapter.quote('DATA')), ['currency'], ['currency']) }} as currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('nested_stream_with_co___long_names_partition') }} as table_alias --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA -{{ cross_join_unnest('partition', adapter.quote('DATA')) }} -where 1 = 1 -and {{ adapter.quote('DATA') }} is not null -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index f4b9f7bb2f07..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,162 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - post_hook = [" - {% - set final_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='nested_stream_with_co__lting_into_long_names' - ) - %} - {# - If the final table doesn't exist, then obviously we can't delete anything from it. - Also, after a reset, the final table is created without the _airbyte_unique_key column (this column is created during the first sync) - So skip this deletion if the column doesn't exist. (in this case, the table is guaranteed to be empty anyway) - #} - {% - if final_table_relation is not none and '_airbyte_unique_key' in adapter.get_columns_in_relation(final_table_relation)|map(attribute='name') - %} - -- Delete records which are no longer active: - -- This query is equivalent, but the left join version is more performant: - -- delete from final_table where unique_key in ( - -- select unique_key from scd_table where 1 = 1 - -- ) and unique_key not in ( - -- select unique_key from scd_table where active_row = 1 - -- ) - -- We're incremental against normalized_at rather than emitted_at because we need to fetch the SCD - -- entries that were _updated_ recently. This is because a deleted record will have an SCD record - -- which was emitted a long time ago, but recently re-normalized to have active_row = 0. - delete from {{ final_table_relation }} where {{ final_table_relation }}._airbyte_unique_key in ( - select recent_records.unique_key - from ( - select distinct _airbyte_unique_key as unique_key - from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} - ) recent_records - left join ( - select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count - from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('nested_stream_with_co__lting_into_long_names')) }} - group by _airbyte_unique_key - ) active_counts - on recent_records.unique_key = active_counts.unique_key - where active_count is null or active_count = 0 - ) - {% else %} - -- We have to have a non-empty query, so just do a noop delete - delete from {{ this }} where 1=0 - {% endif %} - ","drop view _airbyte_test_normalization.nested_stream_with_co_1g_into_long_names_stg"], - tags = [ "top-level" ] -) }} --- depends_on: ref('nested_stream_with_co_1g_into_long_names_stg') -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('nested_stream_with_co_1g_into_long_names_stg') }} - -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at', this) }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key - from new_data -), -empty_new_data as ( - -- build an empty table to only keep the table's column types - select * from new_data where 1 = 0 -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('nested_stream_with_co_1g_into_long_names_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) - left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_stg')) }} from new_data - union all - select {{ dbt_utils.star(ref('nested_stream_with_co_1g_into_long_names_stg')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('nested_stream_with_co_1g_into_long_names_stg') }} - -- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - ]) }} as _airbyte_unique_key, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id - order by - {{ adapter.quote('date') }} is null asc, - {{ adapter.quote('date') }} desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when row_number() over ( - partition by id - order by - {{ adapter.quote('date') }} is null asc, - {{ adapter.quote('date') }} desc, - _airbyte_emitted_at desc - ) = 1 then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_nested_strea__nto_long_names_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by - _airbyte_unique_key, - _airbyte_start_at, - _airbyte_emitted_at - order by _airbyte_active_row desc, _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_nested_strea__nto_long_names_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 0c8adc779de9..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,19 +0,0 @@ -{{ config( - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model --- depends_on: {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} -select - _airbyte_nested_strea__nto_long_names_hashid, - double_array_data, - {{ adapter.quote('DATA') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_partition_hashid -from {{ ref('nested_stream_with_co_2g_names_partition_ab3') }} --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 92e44abc9298..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model --- depends_on: {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} -select - _airbyte_partition_hashid, - currency, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_data_hashid -from {{ ref('nested_stream_with_co_3es_partition_data_ab3') }} --- DATA at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 6a17d6258b3e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,18 +0,0 @@ -{{ config( - schema = "test_normalization", - tags = [ "nested" ] -) }} --- Final base SQL model --- depends_on: {{ ref('nested_stream_with_co_3double_array_data_ab3') }} -select - _airbyte_partition_hashid, - id, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_double_array_data_hashid -from {{ ref('nested_stream_with_co_3double_array_data_ab3') }} --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from {{ ref('nested_stream_with_co___long_names_partition') }} -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index 0ea84390902e..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,22 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model --- depends_on: {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} -select - _airbyte_unique_key, - id, - {{ adapter.quote('date') }}, - {{ adapter.quote('partition') }}, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_nested_strea__nto_long_names_hashid -from {{ ref('nested_stream_with_co_1g_into_long_names_scd') }} --- nested_stream_with_co__lting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_s__lting_into_long_names') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/sources.yml deleted file mode 100644 index 50def309c8c4..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/models/generated/sources.yml +++ /dev/null @@ -1,23 +0,0 @@ -version: 2 -sources: -- name: test_normalization - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_arrays - - name: _airbyte_raw_conflict_stream_array - - name: _airbyte_raw_conflict_stream_name - - name: _airbyte_raw_conflict_stream_scalar - - name: _airbyte_raw_nested_s__lting_into_long_names - - name: _airbyte_raw_non_nest__lting_into_long_names - - name: _airbyte_raw_some_stream_that_was_empty - - name: _airbyte_raw_unnest_alias -- name: test_normalization_namespace - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_simple_s__lting_into_long_names diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql deleted file mode 100644 index e755a1afc193..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_co_1g_into_long_names_scd.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`nested_stream_with_co_1g_into_long_names_scd` (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_strea__nto_long_names_hashid`) - ( - select `_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `date`, `partition`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_strea__nto_long_names_hashid` - from test_normalization.`nested_stream_with_co_1g_into_long_names_scd__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql deleted file mode 100644 index 81c83857e4f8..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___long_names_partition.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`nested_stream_with_co___long_names_partition` (`_airbyte_nested_strea__nto_long_names_hashid`, `double_array_data`, `DATA`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_partition_hashid`) - ( - select `_airbyte_nested_strea__nto_long_names_hashid`, `double_array_data`, `DATA`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_partition_hashid` - from test_normalization.`nested_stream_with_co___long_names_partition__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql deleted file mode 100644 index 4aefae0a0267..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`nested_stream_with_co___names_partition_data` (`_airbyte_partition_hashid`, `currency`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_data_hashid`) - ( - select `_airbyte_partition_hashid`, `currency`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_data_hashid` - from test_normalization.`nested_stream_with_co___names_partition_data__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql deleted file mode 100644 index 48aabbfa4a77..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`nested_stream_with_co__ion_double_array_data` (`_airbyte_partition_hashid`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_double_array_data_hashid`) - ( - select `_airbyte_partition_hashid`, `id`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_double_array_data_hashid` - from test_normalization.`nested_stream_with_co__ion_double_array_data__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql deleted file mode 100644 index 331e7b37cd99..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__lting_into_long_names.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`nested_stream_with_co__lting_into_long_names` (`_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_strea__nto_long_names_hashid`) - ( - select `_airbyte_unique_key`, `id`, `date`, `partition`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_nested_strea__nto_long_names_hashid` - from test_normalization.`nested_stream_with_co__lting_into_long_names__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/dbt_project.yml deleted file mode 100755 index 7321f340e62c..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/dbt_project.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: airbyte_utils -version: '1.0' -config-version: 2 -profile: normalize -model-paths: -- models -docs-paths: -- docs -analysis-paths: -- analysis -test-paths: -- tests -seed-paths: -- data -macro-paths: -- macros -target-path: ../build -log-path: ../logs -packages-install-path: /dbt -clean-targets: -- build -- dbt_modules -quoting: - database: true - schema: false - identifier: true -models: - airbyte_utils: - +materialized: table - generated: - airbyte_ctes: - +tags: airbyte_internal_cte - +materialized: ephemeral - airbyte_incremental: - +tags: incremental_tables - +materialized: incremental - airbyte_tables: - +tags: normalized_tables - +materialized: table - airbyte_views: - +tags: airbyte_internal_views - +materialized: view -vars: - dbt_utils_dispatch_list: - - airbyte_utils - json_column: _airbyte_data - models_to_source: - exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization._airbyte_raw_exchange_rate - exchange_rate: test_normalization._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization._airbyte_raw_dedup_cdc_excluded - pos_dedup_cdcx_ab1: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_ab2: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_stg: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_scd: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx: test_normalization._airbyte_raw_pos_dedup_cdcx - 1_prefix_startwith_number_ab1: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_ab2: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_stg: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_scd: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number: test_normalization._airbyte_raw_1_prefix_startwith_number - multiple_column_names_conflicts_ab1: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_ab2: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts - types_testing_ab1: test_normalization._airbyte_raw_types_testing - types_testing_ab2: test_normalization._airbyte_raw_types_testing - types_testing_stg: test_normalization._airbyte_raw_types_testing - types_testing_scd: test_normalization._airbyte_raw_types_testing - types_testing: test_normalization._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index 74ac8045f1ca..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 2dd5d71c8738..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql deleted file mode 100644 index f32d666a8630..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,112 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_stg__dbt_tmp` as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate -select - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) - ) as id, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."currency"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."currency"')) - ) as currency, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."date"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."date"')) - ) as `date`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."timestamp_col"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."timestamp_col"')) - ) as timestamp_col, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD@spéçiäl & characters"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD@spéçiäl & characters"')) - ) as `HKD@spéçiäl & characters`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD_special___characters"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD_special___characters"')) - ) as hkd_special___characters, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."NZD"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."NZD"')) - ) as nzd, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."USD"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."USD"')) - ) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__dedup_exchange_rate_ab1 -select - cast(id as - signed -) as id, - cast(currency as char(1000)) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char(1000)) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char(1000)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__dedup_exchange_rate_ab2 -select - md5(cast(concat(coalesce(cast(id as char(1000)), ''), '-', coalesce(cast(currency as char(1000)), ''), '-', coalesce(cast(`date` as char(1000)), ''), '-', coalesce(cast(timestamp_col as char(1000)), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char(1000)), ''), '-', coalesce(cast(hkd_special___characters as char(1000)), ''), '-', coalesce(cast(nzd as char(1000)), ''), '-', coalesce(cast(usd as char(1000)), '')) as char(1000))) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -where 1 = 1 - - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql deleted file mode 100644 index f1e29571d47b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql +++ /dev/null @@ -1,103 +0,0 @@ - - create view _airbyte_test_normalization.`multiple_column_names_conflicts_stg__dbt_tmp` as ( - -with __dbt__cte__multiple_column_names_conflicts_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: test_normalization._airbyte_raw_multiple_column_names_conflicts -select - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) - ) as id, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User Id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User Id"')) - ) as `User Id`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."user_id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."user_id"')) - ) as user_id, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User id"')) - ) as `User id_1`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."user id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."user id"')) - ) as `user id_2`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User@Id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."User@Id"')) - ) as `User@Id`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."UserId"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."UserId"')) - ) as userid, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from test_normalization._airbyte_raw_multiple_column_names_conflicts as table_alias --- multiple_column_names_conflicts -where 1 = 1 - -), __dbt__cte__multiple_column_names_conflicts_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__multiple_column_names_conflicts_ab1 -select - cast(id as - signed -) as id, - cast(`User Id` as char(1000)) as `User Id`, - cast(user_id as - float -) as user_id, - cast(`User id_1` as - float -) as `User id_1`, - cast(`user id_2` as - float -) as `user id_2`, - cast(`User@Id` as char(1000)) as `User@Id`, - cast(userid as - float -) as userid, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from __dbt__cte__multiple_column_names_conflicts_ab1 --- multiple_column_names_conflicts -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__multiple_column_names_conflicts_ab2 -select - md5(cast(concat(coalesce(cast(id as char(1000)), ''), '-', coalesce(cast(`User Id` as char(1000)), ''), '-', coalesce(cast(user_id as char(1000)), ''), '-', coalesce(cast(`User id_1` as char(1000)), ''), '-', coalesce(cast(`user id_2` as char(1000)), ''), '-', coalesce(cast(`User@Id` as char(1000)), ''), '-', coalesce(cast(userid as char(1000)), '')) as char(1000))) as _airbyte_multiple_col__ames_conflicts_hashid, - tmp.* -from __dbt__cte__multiple_column_names_conflicts_ab2 tmp --- multiple_column_names_conflicts -where 1 = 1 - - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql deleted file mode 100644 index 670db0869ae2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -select - {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, - {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, - {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as {{ adapter.quote('date') }}, - {{ json_extract_scalar('_airbyte_data', ['timestamp_col'], ['timestamp_col']) }} as timestamp_col, - {{ json_extract_scalar('_airbyte_data', ['HKD@spéçiäl & characters'], ['HKD@spéçiäl & characters']) }} as {{ adapter.quote('HKD@spéçiäl & characters') }}, - {{ json_extract_scalar('_airbyte_data', ['HKD_special___characters'], ['HKD_special___characters']) }} as hkd_special___characters, - {{ json_extract_scalar('_airbyte_data', ['NZD'], ['NZD']) }} as nzd, - {{ json_extract_scalar('_airbyte_data', ['USD'], ['USD']) }} as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias --- dedup_exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql deleted file mode 100644 index 8c26a8139c26..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ /dev/null @@ -1,27 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: {{ ref('dedup_exchange_rate_ab1') }} -select - cast(id as {{ dbt_utils.type_bigint() }}) as id, - cast(currency as {{ dbt_utils.type_string() }}) as currency, - case when {{ adapter.quote('date') }} = '' then NULL - else cast({{ adapter.quote('date') }} as date) - end as {{ adapter.quote('date') }} - , - cast({{ empty_string_to_null('timestamp_col') }} as {{ type_timestamp_with_timezone() }}) as timestamp_col, - cast({{ adapter.quote('HKD@spéçiäl & characters') }} as {{ dbt_utils.type_float() }}) as {{ adapter.quote('HKD@spéçiäl & characters') }}, - cast(hkd_special___characters as {{ dbt_utils.type_string() }}) as hkd_special___characters, - cast(nzd as {{ dbt_utils.type_float() }}) as nzd, - cast(usd as {{ dbt_utils.type_float() }}) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at -from {{ ref('dedup_exchange_rate_ab1') }} --- dedup_exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 499ae700143f..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,176 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", - post_hook = [" - {% - set final_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='dedup_exchange_rate' - ) - %} - {# - If the final table doesn't exist, then obviously we can't delete anything from it. - Also, after a reset, the final table is created without the _airbyte_unique_key column (this column is created during the first sync) - So skip this deletion if the column doesn't exist. (in this case, the table is guaranteed to be empty anyway) - #} - {% - if final_table_relation is not none and '_airbyte_unique_key' in adapter.get_columns_in_relation(final_table_relation)|map(attribute='name') - %} - -- Delete records which are no longer active: - -- This query is equivalent, but the left join version is more performant: - -- delete from final_table where unique_key in ( - -- select unique_key from scd_table where 1 = 1 - -- ) and unique_key not in ( - -- select unique_key from scd_table where active_row = 1 - -- ) - -- We're incremental against normalized_at rather than emitted_at because we need to fetch the SCD - -- entries that were _updated_ recently. This is because a deleted record will have an SCD record - -- which was emitted a long time ago, but recently re-normalized to have active_row = 0. - delete from {{ final_table_relation }} where {{ final_table_relation }}._airbyte_unique_key in ( - select recent_records.unique_key - from ( - select distinct _airbyte_unique_key as unique_key - from {{ this }} - where 1=1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} - ) recent_records - left join ( - select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count - from {{ this }} - where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', adapter.quote(this.schema) + '.' + adapter.quote('dedup_exchange_rate')) }} - group by _airbyte_unique_key - ) active_counts - on recent_records.unique_key = active_counts.unique_key - where active_count is null or active_count = 0 - ) - {% else %} - -- We have to have a non-empty query, so just do a noop delete - delete from {{ this }} where 1=0 - {% endif %} - ","drop view _airbyte_test_normalization.dedup_exchange_rate_stg"], - tags = [ "top-level" ] -) }} --- depends_on: ref('dedup_exchange_rate_stg') -with -{% if is_incremental() %} -new_data as ( - -- retrieve incremental "new" data - select - * - from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} - where 1 = 1 - {{ incremental_clause('_airbyte_emitted_at', this) }} -), -new_data_ids as ( - -- build a subset of _airbyte_unique_key from rows that are new - select distinct - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', - ]) }} as _airbyte_unique_key - from new_data -), -empty_new_data as ( - -- build an empty table to only keep the table's column types - select * from new_data where 1 = 0 -), -previous_active_scd_data as ( - -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes - select - {{ star_intersect(ref('dedup_exchange_rate_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} - from {{ this }} as this_data - -- make a join with new_data using primary key to filter active data that need to be updated only - join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key - -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) - left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id - where _airbyte_active_row = 1 -), -input_data as ( - select {{ dbt_utils.star(ref('dedup_exchange_rate_stg')) }} from new_data - union all - select {{ dbt_utils.star(ref('dedup_exchange_rate_stg')) }} from previous_active_scd_data -), -{% else %} -input_data as ( - select * - from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -), -{% endif %} -scd_data as ( - -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key - select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - 'nzd', - ]) }} as _airbyte_unique_key, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('date') }} as _airbyte_start_at, - lag({{ adapter.quote('date') }}) over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by - {{ adapter.quote('date') }} is null asc, - {{ adapter.quote('date') }} desc, - _airbyte_emitted_at desc - ) as _airbyte_end_at, - case when row_number() over ( - partition by id, currency, cast(nzd as {{ dbt_utils.type_string() }}) - order by - {{ adapter.quote('date') }} is null asc, - {{ adapter.quote('date') }} desc, - _airbyte_emitted_at desc - ) = 1 then 1 else 0 end as _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - _airbyte_dedup_exchange_rate_hashid - from input_data -), -dedup_data as ( - select - -- we need to ensure de-duplicated rows for merge/update queries - -- additionally, we generate a unique key for the scd table - row_number() over ( - partition by - _airbyte_unique_key, - _airbyte_start_at, - _airbyte_emitted_at - order by _airbyte_active_row desc, _airbyte_ab_id - ) as _airbyte_row_num, - {{ dbt_utils.surrogate_key([ - '_airbyte_unique_key', - '_airbyte_start_at', - '_airbyte_emitted_at' - ]) }} as _airbyte_unique_key_scd, - scd_data.* - from scd_data -) -select - _airbyte_unique_key, - _airbyte_unique_key_scd, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_start_at, - _airbyte_end_at, - _airbyte_active_row, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_exchange_rate_hashid -from dedup_data where _airbyte_row_num = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index dd4432bd60a5..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,27 +0,0 @@ -{{ config( - unique_key = "_airbyte_unique_key", - schema = "test_normalization", - tags = [ "top-level" ] -) }} --- Final base SQL model --- depends_on: {{ ref('dedup_exchange_rate_scd') }} -select - _airbyte_unique_key, - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_dedup_exchange_rate_hashid -from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} -where 1 = 1 -and _airbyte_active_row = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index bc56f1c5a3cc..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,46 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "test_normalization", - post_hook = [" - {% - set scd_table_relation = adapter.get_relation( - database=this.database, - schema=this.schema, - identifier='exchange_rate_scd' - ) - %} - {% - if scd_table_relation is not none - %} - {% - do adapter.drop_relation(scd_table_relation) - %} - {% endif %} - "], - tags = [ "top-level" ] -) }} --- Final base SQL model --- depends_on: {{ ref('exchange_rate_ab3') }} -select - id, - currency, - {{ adapter.quote('date') }}, - timestamp_col, - {{ adapter.quote('HKD@spéçiäl & characters') }}, - hkd_special___characters, - nzd, - usd, - {{ adapter.quote('column__\'with"_quotes') }}, - datetime_tz, - datetime_no_tz, - time_tz, - time_no_tz, - property_binary_data, - _airbyte_ab_id, - _airbyte_emitted_at, - {{ current_timestamp() }} as _airbyte_normalized_at, - _airbyte_exchange_rate_hashid -from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} -where 1 = 1 - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql deleted file mode 100644 index 86ec2c9e8b1b..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,24 +0,0 @@ -{{ config( - unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", - tags = [ "top-level-intermediate" ] -) }} --- SQL model to build a hash column based on the values of this record --- depends_on: {{ ref('dedup_exchange_rate_ab2') }} -select - {{ dbt_utils.surrogate_key([ - 'id', - 'currency', - adapter.quote('date'), - 'timestamp_col', - adapter.quote('HKD@spéçiäl & characters'), - 'hkd_special___characters', - 'nzd', - 'usd', - ]) }} as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from {{ ref('dedup_exchange_rate_ab2') }} tmp --- dedup_exchange_rate -where 1 = 1 -{{ incremental_clause('_airbyte_emitted_at', this) }} - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/sources.yml deleted file mode 100644 index f51802427655..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/models/generated/sources.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 2 -sources: -- name: test_normalization - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_1_prefix_startwith_number - - name: _airbyte_raw_dedup_cdc_excluded - - name: _airbyte_raw_dedup_exchange_rate - - name: _airbyte_raw_exchange_rate - - name: _airbyte_raw_multiple_column_names_conflicts - - name: _airbyte_raw_pos_dedup_cdcx - - name: _airbyte_raw_renamed_dedup_cdc_excluded - - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql deleted file mode 100644 index 77b40906d7df..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`dedup_exchange_rate_scd` (`_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `currency`, `date`, `timestamp_col`, `HKD@spéçiäl & characters`, `hkd_special___characters`, `nzd`, `usd`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) - ( - select `_airbyte_unique_key`, `_airbyte_unique_key_scd`, `id`, `currency`, `date`, `timestamp_col`, `HKD@spéçiäl & characters`, `hkd_special___characters`, `nzd`, `usd`, `_airbyte_start_at`, `_airbyte_end_at`, `_airbyte_active_row`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid` - from test_normalization.`dedup_exchange_rate_scd__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql deleted file mode 100644 index d67b7a41aba1..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ /dev/null @@ -1,7 +0,0 @@ - - insert into test_normalization.`dedup_exchange_rate` (`_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD@spéçiäl & characters`, `hkd_special___characters`, `nzd`, `usd`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid`) - ( - select `_airbyte_unique_key`, `id`, `currency`, `date`, `timestamp_col`, `HKD@spéçiäl & characters`, `hkd_special___characters`, `nzd`, `usd`, `_airbyte_ab_id`, `_airbyte_emitted_at`, `_airbyte_normalized_at`, `_airbyte_dedup_exchange_rate_hashid` - from test_normalization.`dedup_exchange_rate__dbt_tmp` - ) - \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql deleted file mode 100644 index 2dd5d71c8738..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql deleted file mode 100644 index f32d666a8630..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/tidb/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ /dev/null @@ -1,112 +0,0 @@ - - create view _airbyte_test_normalization.`dedup_exchange_rate_stg__dbt_tmp` as ( - -with __dbt__cte__dedup_exchange_rate_ab1 as ( - --- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate -select - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."id"')) - ) as id, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."currency"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."currency"')) - ) as currency, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."date"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."date"')) - ) as `date`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."timestamp_col"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."timestamp_col"')) - ) as timestamp_col, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD@spéçiäl & characters"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD@spéçiäl & characters"')) - ) as `HKD@spéçiäl & characters`, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD_special___characters"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."HKD_special___characters"')) - ) as hkd_special___characters, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."NZD"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."NZD"')) - ) as nzd, - IF( - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."USD"')) = 'null', - NULL, - JSON_UNQUOTE(JSON_EXTRACT(_airbyte_data, - '$."USD"')) - ) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from test_normalization._airbyte_raw_dedup_exchange_rate as table_alias --- dedup_exchange_rate -where 1 = 1 - -), __dbt__cte__dedup_exchange_rate_ab2 as ( - --- SQL model to cast each column to its adequate SQL type converted from the JSON schema type --- depends_on: __dbt__cte__dedup_exchange_rate_ab1 -select - cast(id as - signed -) as id, - cast(currency as char(1000)) as currency, - case when `date` = '' then NULL - else cast(`date` as date) - end as `date` - , - cast(nullif(timestamp_col, '') as char(1000)) as timestamp_col, - cast(`HKD@spéçiäl & characters` as - float -) as `HKD@spéçiäl & characters`, - cast(hkd_special___characters as char(1000)) as hkd_special___characters, - cast(nzd as - float -) as nzd, - cast(usd as - float -) as usd, - _airbyte_ab_id, - _airbyte_emitted_at, - current_timestamp() as _airbyte_normalized_at -from __dbt__cte__dedup_exchange_rate_ab1 --- dedup_exchange_rate -where 1 = 1 - -)-- SQL model to build a hash column based on the values of this record --- depends_on: __dbt__cte__dedup_exchange_rate_ab2 -select - md5(cast(concat(coalesce(cast(id as char(1000)), ''), '-', coalesce(cast(currency as char(1000)), ''), '-', coalesce(cast(`date` as char(1000)), ''), '-', coalesce(cast(timestamp_col as char(1000)), ''), '-', coalesce(cast(`HKD@spéçiäl & characters` as char(1000)), ''), '-', coalesce(cast(hkd_special___characters as char(1000)), ''), '-', coalesce(cast(nzd as char(1000)), ''), '-', coalesce(cast(usd as char(1000)), '')) as char(1000))) as _airbyte_dedup_exchange_rate_hashid, - tmp.* -from __dbt__cte__dedup_exchange_rate_ab2 tmp --- dedup_exchange_rate -where 1 = 1 - - ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/destination_catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/destination_catalog.json index 8fc21bb20955..11fcc6819fb1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/destination_catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/destination_catalog.json @@ -8,22 +8,22 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "is_default": { - "$ref": "WellKnownTypes.json#/definitions/Boolean" + "type": ["null", "boolean"] }, "species": { "type": ["null", "object"], "properties": { "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -33,10 +33,10 @@ "type": ["null", "object"], "properties": { "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages1.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages1.txt index f10511278564..0238a2863316 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages1.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages1.txt @@ -1,4 +1,4 @@ -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"132","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508770}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"133","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508771}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"134","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508772}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"135","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508773}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":132,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508770}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":133,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508771}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":134,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508772}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":135,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508773}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages2.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages2.txt index eed530ebc214..b5b4385b2d56 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages2.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/redshift_normalization_migration/messages2.txt @@ -1,4 +1,4 @@ -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"132","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508774}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"133","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508775}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"134","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508776}} -{"type":"RECORD","record":{"stream":"pokemon","data":{"id":"135","name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508777}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":132,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508774}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":133,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508775}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":134,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508776}} +{"type":"RECORD","record":{"stream":"pokemon","data":{"id":135,"name":"ditto","is_default":true,"species":{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-species/132/"},"forms":[{"name":"ditto","url":"https://pokeapi.co/api/v2/pokemon-form/132/"}]},"emitted_at":1650266508777}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json index 6119090daa1d..4e5105f136e0 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/catalog.json @@ -7,13 +7,10 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "date": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "partition": { "type": ["null", "object"], @@ -25,7 +22,7 @@ "items": { "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } @@ -36,7 +33,7 @@ "items": { "properties": { "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } @@ -61,13 +58,10 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "date": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -86,13 +80,10 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "date": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -113,13 +104,10 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "date": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -138,10 +126,7 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "conflict_stream_name": { "type": ["null", "object"], @@ -152,14 +137,14 @@ "type": "object", "properties": { "groups": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } }, "custom_fields": { "items": { "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] }, "value": {} }, @@ -168,7 +153,7 @@ "type": ["null", "array"] }, "conflict_stream_name": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" } } } @@ -191,13 +176,10 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "conflict_stream_scalar": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" } } }, @@ -216,10 +198,7 @@ "type": ["null", "object"], "properties": { "id": { - "oneOf": [ - { "$ref": "WellKnownTypes.json#/definitions/Number" }, - { "$ref": "WellKnownTypes.json#/definitions/String" } - ] + "type": ["null", "number", "string"] }, "conflict_stream_array": { "type": ["null", "array"], @@ -229,7 +208,7 @@ "items": { "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } } @@ -253,7 +232,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "children": { "type": ["null", "array"], @@ -261,20 +240,20 @@ "type": "object", "properties": { "ab_id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] }, "owner": { "type": ["null", "object"], "properties": { "owner_id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] }, "column`_'with\"_quotes": { "type": ["null", "array"], "items": { "properties": { "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } @@ -303,7 +282,7 @@ "array_of_strings": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } }, "nested_array_parent": { @@ -312,7 +291,7 @@ "nested_array": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt index 1c290c079a74..e349c09afc31 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages.txt @@ -1,18 +1,18 @@ -{"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599000, "data": { "id": "4.2", "date": "2020-08-29T00:00:00Z", "partition": { "double_array_data": [[ { "id": "EUR" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} +{"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599000, "data": { "id": 4.2, "date": "2020-08-29T00:00:00Z", "partition": { "double_array_data": [[ { "id": "EUR" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} {"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599100, "data": { "id": "test record", "date": "2020-08-31T00:00:00Z", "partition": { "double_array_data": [[ { "id": "USD" } ], [ { "id": "GBP" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} -{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":"1","conflict_stream_name":{"conflict_stream_name": {"groups": "1", "custom_fields": [{"id":"1", "value":"3"}, {"id":"2", "value":"4"}], "conflict_stream_name": "3"}}},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":"2","conflict_stream_name":{"conflict_stream_name": {"groups": "2", "custom_fields": [{"id":"1", "value":"3"}, {"id":"2", "value":"4"}], "conflict_stream_name": "3"}}},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":1,"conflict_stream_name":{"conflict_stream_name": {"groups": "1", "custom_fields": [{"id":1, "value":3}, {"id":2, "value":4}], "conflict_stream_name": 3}}},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":2,"conflict_stream_name":{"conflict_stream_name": {"groups": "2", "custom_fields": [{"id":1, "value":3}, {"id":2, "value":4}], "conflict_stream_name": 3}}},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"1","conflict_stream_scalar": "2"},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"2","conflict_stream_scalar": "2"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":1,"conflict_stream_scalar": 2},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":2,"conflict_stream_scalar": 2},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":"1", "conflict_stream_array": {"conflict_stream_array": [{"id": "1"}, {"id": "2"}, {"id": "3"}]}}, "emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":"2", "conflict_stream_array": {"conflict_stream_array": [{"id": "4"}, {"id": "5"}, {"id": "6"}]}}, "emitted_at":1623861860}} +{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":1, "conflict_stream_array": {"conflict_stream_array": [{"id": 1}, {"id": 2}, {"id": 3}]}}, "emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":2, "conflict_stream_array": {"conflict_stream_array": [{"id": 4}, {"id": 5}, {"id": 6}]}}, "emitted_at":1623861860}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"1","conflict_stream_scalar": "2"},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"2","conflict_stream_scalar": "2"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":1,"conflict_stream_scalar": 2},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":2,"conflict_stream_scalar": 2},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":"1", "children": [{"ab_id": "1", "owner": {"owner_id": "1", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": "2", "owner": {"owner_id": "2", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":"2", "children": [{"ab_id": "3", "owner": {"owner_id": "3", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": "4", "owner": {"owner_id": "4", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":1, "children": [{"ab_id": 1, "owner": {"owner_id": 1, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 2, "owner": {"owner_id": 2, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":2, "children": [{"ab_id": 3, "owner": {"owner_id": 3, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 4, "owner": {"owner_id": 4, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} {"type":"RECORD","record":{"stream":"arrays","emitted_at":1602638599000,"data":{"array_of_strings":["string1",null,"string2","string3"],"nested_array_parent":{"nested_array":["string1",null,"string2"]}}}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages_incremental.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages_incremental.txt index c7b4bcfec5f7..ae1cf0f5c0b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages_incremental.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_nested_streams/data_input/messages_incremental.txt @@ -1,22 +1,22 @@ -{"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599000, "data": { "id": "4.2", "date": "2020-08-29T00:00:00Z", "partition": { "double_array_data": [[ { "id": "EUR" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} +{"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599000, "data": { "id": 4.2, "date": "2020-08-29T00:00:00Z", "partition": { "double_array_data": [[ { "id": "EUR" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} {"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638599100, "data": { "id": "test record", "date": "2020-08-31T00:00:00Z", "partition": { "double_array_data": [[ { "id": "USD" } ], [ { "id": "GBP" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} {"type": "RECORD", "record": {"stream": "nested_stream_with_complex_columns_resulting_into_long_names", "emitted_at": 1602638600000, "data": { "id": "new record", "date": "2020-09-10T00:00:00Z", "partition": { "double_array_data": [[ { "id": "GBP" } ], [ { "id": "HKD" } ]], "DATA": [ {"currency": "EUR" } ], "column`_'with\"_quotes": [ {"currency": "EUR" } ] } }}} -{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":"1","conflict_stream_name":{"conflict_stream_name": {"groups": "1", "custom_fields": [{"id":"1", "value":3}, {"id":"2", "value":4}], "conflict_stream_name": "3"}}},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":"2","conflict_stream_name":{"conflict_stream_name": {"groups": "2", "custom_fields": [{"id":"1", "value":3}, {"id":"2", "value":4}], "conflict_stream_name": "3"}}},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":1,"conflict_stream_name":{"conflict_stream_name": {"groups": "1", "custom_fields": [{"id":1, "value":3}, {"id":2, "value":4}], "conflict_stream_name": 3}}},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_name","data":{"id":2,"conflict_stream_name":{"conflict_stream_name": {"groups": "2", "custom_fields": [{"id":1, "value":3}, {"id":2, "value":4}], "conflict_stream_name": 3}}},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"1","conflict_stream_scalar": "2"},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"2","conflict_stream_scalar": "2"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":1,"conflict_stream_scalar": 2},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":2,"conflict_stream_scalar": 2},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":"1", "conflict_stream_array": {"conflict_stream_array": [{"id": "1"}, {"id": "2"}, {"id": "3"}]}}, "emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":"2", "conflict_stream_array": {"conflict_stream_array": [{"id": "4"}, {"id": "5"}, {"id": "6"}]}}, "emitted_at":1623861860}} +{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":1, "conflict_stream_array": {"conflict_stream_array": [{"id": 1}, {"id": 2}, {"id": 3}]}}, "emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_array","data":{"id":2, "conflict_stream_array": {"conflict_stream_array": [{"id": 4}, {"id": 5}, {"id": 6}]}}, "emitted_at":1623861860}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"1","conflict_stream_scalar": "2"},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":"2","conflict_stream_scalar": "2"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":1,"conflict_stream_scalar": 2},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"conflict_stream_scalar","data":{"id":2,"conflict_stream_scalar": 2},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":"1", "children": [{"ab_id": "1", "owner": {"owner_id": "1", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": "2", "owner": {"owner_id": "2", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":"2", "children": [{"ab_id": "3", "owner": {"owner_id": "3", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": "4", "owner": {"owner_id": "4", "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":1, "children": [{"ab_id": 1, "owner": {"owner_id": 1, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 2, "owner": {"owner_id": 2, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"unnest_alias","data":{"id":2, "children": [{"ab_id": 3, "owner": {"owner_id": 3, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}},{"ab_id": 4, "owner": {"owner_id": 4, "column`_'with\"_quotes": [ {"currency": "EUR" } ]}}]},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":"1","date": "2020-11-05"},"emitted_at":1623871660}} -{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":"2","date": "2020-11-06"},"emitted_at":1623872660}} -{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":"3","date": "2020-11-06"},"emitted_at":1623873660}} +{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":1,"date": "2020-11-05"},"emitted_at":1623871660}} +{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":2,"date": "2020-11-06"},"emitted_at":1623872660}} +{"type":"RECORD","record":{"stream":"some_stream_that_was_empty","data":{"id":3,"date": "2020-11-06"},"emitted_at":1623873660}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog.json index db7721083733..37d6c7d9a939 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog.json @@ -7,25 +7,29 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "datetime_to_string": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "string_to_dt": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "number_to_int": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "int_to_number": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" } } }, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_incremental.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_incremental.json index 524761cddb67..04b78b4b435f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_incremental.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_incremental.json @@ -7,25 +7,29 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "datetime_to_string": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "string_to_dt": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "number_to_int": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "int_to_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" } } }, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_reset.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_reset.json index 08b499629be2..9a76b76cda8b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_reset.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_catalog_reset.json @@ -7,25 +7,29 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "datetime_to_string": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "string_to_dt": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "number_to_int": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "int_to_number": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" } } }, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_messages.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_messages.txt index 163e7a966826..e35685cb629a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_messages.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_drop_scd_messages.txt @@ -1,5 +1,5 @@ -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637589000, "data": { "id": "1", "date": "2022-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "datetime_to_string":"2022-10-01T01:04:04-04:00", "string_to_dt":"2022-11-01T02:03:04-07:00", "number_to_int": "1", "int_to_number": "10"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637689100, "data": { "id": "2", "date": "2022-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "datetime_to_string":"2022-10-02T01:04:04-04:00", "string_to_dt":"2022-11-02T03:04:05-07:00", "number_to_int": "10", "int_to_number": "11"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637789200, "data": { "id": "3", "date": "2022-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "datetime_to_string":"2022-10-03T01:04:04-04:00", "string_to_dt":"2022-11-03T03:04:06-07:00", "number_to_int": "11", "int_to_number": "12"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637889300, "data": { "id": "4", "date": "2022-09-01", "timestamp_col": "2020-08-31T00:00:00+0000", "datetime_to_string":"2022-10-04T01:04:04-04:00", "string_to_dt":"2022-11-04T03:04:07-07:00", "number_to_int": "111", "int_to_number": "133"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": "5", "date": "2022-09-02", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"2022-10-05T01:04:04-04:00", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": "1010", "int_to_number": "1300"}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637589000, "data": { "id": 1, "date": "2022-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "datetime_to_string":"2022-10-01T01:04:04-04:00", "string_to_dt":"2022-11-01T02:03:04-07:00", "number_to_int": 1, "int_to_number": 10}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637689100, "data": { "id": 2, "date": "2022-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "datetime_to_string":"2022-10-02T01:04:04-04:00", "string_to_dt":"2022-11-02T03:04:05-07:00", "number_to_int": 10, "int_to_number": 11}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637789200, "data": { "id": 3, "date": "2022-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "datetime_to_string":"2022-10-03T01:04:04-04:00", "string_to_dt":"2022-11-03T03:04:06-07:00", "number_to_int": 11, "int_to_number": 12}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637889300, "data": { "id": 4, "date": "2022-09-01", "timestamp_col": "2020-08-31T00:00:00+0000", "datetime_to_string":"2022-10-04T01:04:04-04:00", "string_to_dt":"2022-11-04T03:04:07-07:00", "number_to_int": 111, "int_to_number": 133}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": 5, "date": "2022-09-02", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"2022-10-05T01:04:04-04:00", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": 1010, "int_to_number": 1300}}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_scd_reset_messages_incremental.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_scd_reset_messages_incremental.txt index 985706845fad..492efbaea0ae 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_scd_reset_messages_incremental.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_reset_scd_overwrite/data_input/test_scd_reset_messages_incremental.txt @@ -1,6 +1,6 @@ -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637589000, "data": { "id": "1", "date": "2022-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "datetime_to_string":"2022-10-01T01:04:04-04:00", "string_to_dt":"2022-11-01T02:03:04-07:00", "number_to_int": "1", "int_to_number": "10"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637689100, "data": { "id": "2", "date": "2022-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "datetime_to_string":"2022-10-02T01:04:04-04:00", "string_to_dt":"2022-11-02T03:04:05-07:00", "number_to_int": "10", "int_to_number": "11"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637789200, "data": { "id": "3", "date": "2022-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "datetime_to_string":"2022-10-03T01:04:04-04:00", "string_to_dt":"2022-11-03T03:04:06-07:00", "number_to_int": "11", "int_to_number": "12"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637889300, "data": { "id": "4", "date": "2022-09-01", "timestamp_col": "2020-08-31T00:00:00+0000", "datetime_to_string":"2022-10-04T01:04:04-04:00", "string_to_dt":"2022-11-04T03:04:07-07:00", "number_to_int": "111", "int_to_number": "133"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": "5", "date": "2022-09-02", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"2022-10-05T01:04:04-04:00", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": "1010", "int_to_number": "1300"}}} -{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": "6", "date": "2022-09-03", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"this is a string, not a datetime value", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": "1010", "int_to_number": "1300.25"}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637589000, "data": { "id": 1, "date": "2022-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "datetime_to_string":"2022-10-01T01:04:04-04:00", "string_to_dt":"2022-11-01T02:03:04-07:00", "number_to_int": 1, "int_to_number": 10}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637689100, "data": { "id": 2, "date": "2022-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "datetime_to_string":"2022-10-02T01:04:04-04:00", "string_to_dt":"2022-11-02T03:04:05-07:00", "number_to_int": 10, "int_to_number": 11}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637789200, "data": { "id": 3, "date": "2022-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "datetime_to_string":"2022-10-03T01:04:04-04:00", "string_to_dt":"2022-11-03T03:04:06-07:00", "number_to_int": 11, "int_to_number": 12}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637889300, "data": { "id": 4, "date": "2022-09-01", "timestamp_col": "2020-08-31T00:00:00+0000", "datetime_to_string":"2022-10-04T01:04:04-04:00", "string_to_dt":"2022-11-04T03:04:07-07:00", "number_to_int": 111, "int_to_number": 133}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": 5, "date": "2022-09-02", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"2022-10-05T01:04:04-04:00", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": 1010, "int_to_number": 1300}}} +{"type": "RECORD", "record": {"stream": "stream_test_scd_drop", "emitted_at": 1602637989400, "data": { "id": 6, "date": "2022-09-03", "timestamp_col": "2020-09-01T00:00:00Z", "datetime_to_string":"this is a string, not a datetime value", "string_to_dt":"2022-11-05T03:04:08-12:00", "number_to_int": 1010, "int_to_number": 1300.25}}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json index 4d2b83d2b8e2..daef42fdf220 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog.json @@ -7,46 +7,53 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "HKD@spéçiäl & characters": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "HKD_special___characters": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "NZD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "USD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "column`_'with\"_quotes": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "datetime_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_with_timezone" }, "datetime_no_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" }, "time_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithTimezone" + "type": "string", + "format": "time", + "airbyte_type": "time_with_timezone" }, "time_no_tz": { - "$ref": "WellKnownTypes.json#/definitions/TimeWithoutTimezone" - }, - "property_binary_data": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" + "type": "string", + "format": "time", + "airbyte_type": "time_without_timezone" } } }, @@ -65,28 +72,30 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "HKD@spéçiäl & characters": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "HKD_special___characters": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "NZD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "USD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" } } }, @@ -106,10 +115,10 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "_ab_cdc_updated_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, @@ -129,19 +138,19 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["string", "null"] }, "_ab_cdc_lsn": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_updated_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_deleted_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, @@ -161,22 +170,22 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["string", "null"] }, "_ab_cdc_lsn": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_updated_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_deleted_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_log_pos": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, @@ -196,13 +205,14 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "text": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -222,25 +232,25 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "User Id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["string", "null"] }, "user_id": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "User id": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "user id": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "User@Id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "UserId": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, @@ -260,16 +270,15 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "airbyte_integer_column": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "number", + "airbyte_type": "integer" }, "nullable_airbyte_integer_column": { - "$ref": "WellKnownTypes.json#/definitions/Integer" - }, - "property_binary_data": { - "$ref": "WellKnownTypes.json#/definitions/BinaryData" + "type": ["null", "number"], + "airbyte_type": "integer" } } } diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json index b6bcbb6c3199..a54e89c4ff2e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/catalog_schema_change.json @@ -7,31 +7,33 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "new_column": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "HKD@spéçiäl & characters": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "NZD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "USD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "column`_'with\"_quotes": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -50,28 +52,30 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "currency": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "new_column": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "date": { - "$ref": "WellKnownTypes.json#/definitions/Date" + "type": "string", + "format": "date" }, "timestamp_col": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithTimezone" + "type": "string", + "format": "date-time" }, "HKD@spéçiäl & characters": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "NZD": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": "number" }, "USD": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" } } }, @@ -91,19 +95,19 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["string", "null"] }, "_ab_cdc_lsn": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_updated_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_deleted_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, @@ -123,19 +127,19 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": "integer" }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["string", "null"] }, "_ab_cdc_lsn": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_updated_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] }, "_ab_cdc_deleted_at": { - "$ref": "WellKnownTypes.json#/definitions/Number" + "type": ["null", "number"] } } }, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages.txt index 94f7aab44927..ed52bb6c8674 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages.txt @@ -1,67 +1,67 @@ -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637589000, "data": { "id": "1", "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": "1.14", "HKD@spéçiäl & characters": "2.13", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "property_binary_data": "dGVzdA==" }}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637689100, "data": { "id": "1", "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": "1.14", "HKD@spéçiäl & characters": "7.15", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637789200, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": "3.89", "HKD@spéçiäl & characters": "7.12", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.16"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637889300, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": "1.14", "HKD@spéçiäl & characters": "7.99", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.99"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637989400, "data": { "id": "2", "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "8", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.16"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990700, "data": { "id": "1", "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "1.14", "HKD@spéçiäl & characters": "10.5", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "", "timestamp_col": "", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990900, "data": { "id": "3", "currency": "GBP", "NZD": "3.14", "HKD@spéçiäl & characters": "9.2", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991000, "data": { "id": "2", "currency": "EUR", "NZD": "3.89", "HKD@spéçiäl & characters": "7.02", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991100, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "8.12", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991200, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "9.23", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991300, "data": { "id": "6", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "9.23", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "datetime_tz": "2022-01-14T01:04:04-04:00", "datetime_no_tz": "2022-01-14T01:04:04", "time_tz": "01:04:04-04:00", "time_no_tz": "01:04:04"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a" }}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.99}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637989400, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 8, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990700, "data": { "id": 1, "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 10.5, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "", "timestamp_col": "", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990900, "data": { "id": 3, "currency": "GBP", "NZD": 3.14, "HKD@spéçiäl & characters": 9.2, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991000, "data": { "id": 2, "currency": "EUR", "NZD": 3.89, "HKD@spéçiäl & characters": 7.02, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991100, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 8.12, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991200, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 9.23, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637991300, "data": { "id": 6, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 9.23, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "datetime_tz": "2022-01-14T01:04:04-04:00", "datetime_no_tz": "2022-01-14T01:04:04", "time_tz": "01:04:04-04:00", "time_no_tz": "01:04:04"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637589000, "data": { "id": "1", "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": "1.14", "HKD@spéçiäl & characters": "2.13", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a" }}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637689100, "data": { "id": "1", "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": "1.14", "HKD@spéçiäl & characters": "7.15", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637789200, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": "3.89", "HKD@spéçiäl & characters": "7.12", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.16"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637889300, "data": { "id": "2", "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": "1.14", "HKD@spéçiäl & characters": "7.99", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.99"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637989400, "data": { "id": "2", "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "8", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10.16"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990700, "data": { "id": "1", "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "1.14", "HKD@spéçiäl & characters": "10.5", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "", "timestamp_col": "", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990900, "data": { "id": "3", "currency": "GBP", "NZD": "3.14", "HKD@spéçiäl & characters": "9.2", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991000, "data": { "id": "2", "currency": "EUR", "NZD": "3.89", "HKD@spéçiäl & characters": "7.02", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991100, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "8.12", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991200, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "9.23", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637589000, "data": { "id": 1, "currency": "USD", "date": "2020-08-29", "timestamp_col": "2020-08-29T00:00:00.000000-0000", "NZD": 1.14, "HKD@spéçiäl & characters": 2.13, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a" }}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637689100, "data": { "id": 1, "currency": "USD", "date": "2020-08-30", "timestamp_col": "2020-08-30T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 7.15, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637789200, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+00", "NZD": 3.89, "HKD@spéçiäl & characters": 7.12, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637889300, "data": { "id": 2, "currency": "EUR", "date": "2020-08-31", "timestamp_col": "2020-08-31T00:00:00+0000", "NZD": 1.14, "HKD@spéçiäl & characters": 7.99, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.99}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637989400, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 8, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10.16}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990700, "data": { "id": 1, "currency": "USD", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 1.14, "HKD@spéçiäl & characters": 10.5, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "2020-09-01", "timestamp_col": "2020-09-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "", "timestamp_col": "", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990900, "data": { "id": 3, "currency": "GBP", "NZD": 3.14, "HKD@spéçiäl & characters": 9.2, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991000, "data": { "id": 2, "currency": "EUR", "NZD": 3.89, "HKD@spéçiäl & characters": 7.02, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991100, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 8.12, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637991200, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 9.23, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} # Note that some of the IDs are inserted and then deleted; this should be reflected as a single row in the SCD model with _airbyte_active_row set to 0. -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"1","name":"mazda","_ab_cdc_updated_at":"1623849130530","_ab_cdc_lsn":"26971624","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"2","name":"toyata","_ab_cdc_updated_at":"1623849130549","_ab_cdc_lsn":"26971624","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"4","name":"bmw","_ab_cdc_updated_at":"1623849314535","_ab_cdc_lsn":"26974776","_ab_cdc_deleted_at":null},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"5","name":"vw","_ab_cdc_updated_at":"1623849314663","_ab_cdc_lsn":"26975264","_ab_cdc_deleted_at":null},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"4","name":null,"_ab_cdc_updated_at":"1623849314791","_ab_cdc_lsn":"26975440","_ab_cdc_deleted_at":"1623849314791"},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"6","name":"opel","_ab_cdc_updated_at":"1623850868109","_ab_cdc_lsn":"27009440","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"7","name":"lotus","_ab_cdc_updated_at":"1623850868237","_ab_cdc_lsn":"27010048","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":1,"name":"mazda","_ab_cdc_updated_at":1623849130530,"_ab_cdc_lsn":26971624,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":2,"name":"toyata","_ab_cdc_updated_at":1623849130549,"_ab_cdc_lsn":26971624,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":4,"name":"bmw","_ab_cdc_updated_at":1623849314535,"_ab_cdc_lsn":26974776,"_ab_cdc_deleted_at":null},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":5,"name":"vw","_ab_cdc_updated_at":1623849314663,"_ab_cdc_lsn":26975264,"_ab_cdc_deleted_at":null},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":4,"name":null,"_ab_cdc_updated_at":1623849314791,"_ab_cdc_lsn":26975440,"_ab_cdc_deleted_at":1623849314791},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":6,"name":"opel","_ab_cdc_updated_at":1623850868109,"_ab_cdc_lsn":27009440,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":7,"name":"lotus","_ab_cdc_updated_at":1623850868237,"_ab_cdc_lsn":27010048,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} # messages_incremental.txt has a dedup_cdc_excluded record with emitted_at=1623860160, i.e. older than this record. If you delete/modify this record, make sure to maintain that relationship. -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"6","name":null,"_ab_cdc_updated_at":"1623850868371","_ab_cdc_lsn":"27010232","_ab_cdc_deleted_at":"1623850868371"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":6,"name":null,"_ab_cdc_updated_at":1623850868371,"_ab_cdc_lsn":27010232,"_ab_cdc_deleted_at":1623850868371},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"1","name":"mazda","_ab_cdc_updated_at":"1623849130530","_ab_cdc_lsn":"26971624","_ab_cdc_log_pos": "33274","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"2","name":"toyata","_ab_cdc_updated_at":"1623849130549","_ab_cdc_lsn":"26971624","_ab_cdc_log_pos": "33275","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"2","name":"bmw","_ab_cdc_updated_at":"1623849314535","_ab_cdc_lsn":"26974776","_ab_cdc_log_pos": "33278","_ab_cdc_deleted_at":null},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"3","name":null,"_ab_cdc_updated_at":"1623849314791","_ab_cdc_lsn":"26975440","_ab_cdc_log_pos": "33274","_ab_cdc_deleted_at":"1623849314791"},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"4","name":"lotus","_ab_cdc_updated_at":"1623850868237","_ab_cdc_lsn":"27010048","_ab_cdc_log_pos": "33271","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"4","name":null,"_ab_cdc_updated_at":"1623850868371","_ab_cdc_lsn":"27010232","_ab_cdc_log_pos": "33279","_ab_cdc_deleted_at":"1623850868371"},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"5","name":"lotus","_ab_cdc_updated_at":"1623850868371","_ab_cdc_lsn":"27010048","_ab_cdc_log_pos": "33280","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"5","name":"lily","_ab_cdc_updated_at":"1623850868371","_ab_cdc_lsn":"27010232","_ab_cdc_log_pos": "33281","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":1,"name":"mazda","_ab_cdc_updated_at":1623849130530,"_ab_cdc_lsn":26971624,"_ab_cdc_log_pos": 33274,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":2,"name":"toyata","_ab_cdc_updated_at":1623849130549,"_ab_cdc_lsn":26971624,"_ab_cdc_log_pos": 33275,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":2,"name":"bmw","_ab_cdc_updated_at":1623849314535,"_ab_cdc_lsn":26974776,"_ab_cdc_log_pos": 33278,"_ab_cdc_deleted_at":null},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":3,"name":null,"_ab_cdc_updated_at":1623849314791,"_ab_cdc_lsn":26975440,"_ab_cdc_log_pos": 33274,"_ab_cdc_deleted_at":1623849314791},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":4,"name":"lotus","_ab_cdc_updated_at":1623850868237,"_ab_cdc_lsn":27010048,"_ab_cdc_log_pos": 33271,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":4,"name":null,"_ab_cdc_updated_at":1623850868371,"_ab_cdc_lsn":27010232,"_ab_cdc_log_pos": 33279,"_ab_cdc_deleted_at":1623850868371},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":5,"name":"lotus","_ab_cdc_updated_at":1623850868371,"_ab_cdc_lsn":27010048,"_ab_cdc_log_pos": 33280,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":5,"name":"lily","_ab_cdc_updated_at":1623850868371,"_ab_cdc_lsn":27010232,"_ab_cdc_log_pos": 33281,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637589000, "data": { "id": "1", "date": "2020-08-29", "text": "hi 1"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637689100, "data": { "id": "1", "date": "2020-08-30", "text": "hi 2"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637789200, "data": { "id": "2", "date": "2020-08-31", "text": "hi 1"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637889300, "data": { "id": "2", "date": "2020-08-31", "text": "hi 2"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637989400, "data": { "id": "2", "date": "2020-09-01", "text": "hi 3"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637990700, "data": { "id": "1", "date": "2020-09-01", "text": "hi 3"}}} -{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637990800, "data": { "id": "2", "date": "2020-09-01", "text": "hi 4"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637589000, "data": { "id": 1, "date": "2020-08-29", "text": "hi 1"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637689100, "data": { "id": 1, "date": "2020-08-30", "text": "hi 2"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637789200, "data": { "id": 2, "date": "2020-08-31", "text": "hi 1"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637889300, "data": { "id": 2, "date": "2020-08-31", "text": "hi 2"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637989400, "data": { "id": 2, "date": "2020-09-01", "text": "hi 3"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637990700, "data": { "id": 1, "date": "2020-09-01", "text": "hi 3"}}} +{"type": "RECORD", "record": {"stream": "1_prefix_startwith_number", "emitted_at": 1602637990800, "data": { "id": 2, "date": "2020-09-01", "text": "hi 4"}}} -{"type":"RECORD","record":{"stream":"multiple_column_names_conflicts","data":{"id":"1","User Id":"chris","user_id":"42","User id":"300","user id": "102","UserId": "101"},"emitted_at":1623959926}} +{"type":"RECORD","record":{"stream":"multiple_column_names_conflicts","data":{"id":1,"User Id":"chris","user_id":42,"User id":300,"user id": 102,"UserId":101},"emitted_at":1623959926}} # These records are verified in types_testing_incorrect_values.sql. If you add/remove entries, make sure to update that file as well. # IMPORTANT: big_integer_column and nullable_big_integer_column were removed from catalog.json because of difficulties in implementing NUMERIC support. # This is fine, because no major sources currently produce big_integer fields. # After that functionality is completed, we should restore their entries to catalog.json. # Verify max value for int64, and a 28-digit value for big_integer. (28 is larger than an int64 can handle, but still within bounds for a BigQuery NUMERIC column) -{"type":"RECORD","record":{"stream":"types_testing","data":{"id":"1","airbyte_integer_column":"9223372036854775807","nullable_airbyte_integer_column":"9223372036854775807","big_integer_column":"1234567890123456789012345678","nullable_big_integer_column":"1234567890123456789012345678", "property_binary_data": "dGVzdA=="},"emitted_at":1623959926}} +{"type":"RECORD","record":{"stream":"types_testing","data":{"id":1,"airbyte_integer_column":9223372036854775807,"nullable_airbyte_integer_column":9223372036854775807,"big_integer_column":"1234567890123456789012345678","nullable_big_integer_column":"1234567890123456789012345678"},"emitted_at":1623959926}} # Verify max value for int64, and a negative 28-digit value for big_integer -{"type":"RECORD","record":{"stream":"types_testing","data":{"id":"2","airbyte_integer_column":"-9223372036854775808","nullable_airbyte_integer_column":"-9223372036854775808","big_integer_column":"-1234567890123456789012345678","nullable_big_integer_column":"-1234567890123456789012345678"},"emitted_at":1623959926}} +{"type":"RECORD","record":{"stream":"types_testing","data":{"id":2,"airbyte_integer_column":-9223372036854775808,"nullable_airbyte_integer_column":-9223372036854775808,"big_integer_column":"-1234567890123456789012345678","nullable_big_integer_column":"-1234567890123456789012345678"},"emitted_at":1623959926}} # Verify nullable values -{"type":"RECORD","record":{"stream":"types_testing","data":{"id":"3","airbyte_integer_column":"0","big_integer_column":"0", "property_binary_data": null},"emitted_at":1623959926}} +{"type":"RECORD","record":{"stream":"types_testing","data":{"id":3,"airbyte_integer_column":0,"big_integer_column":0},"emitted_at":1623959926}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_incremental.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_incremental.txt index b7e0fcfeee42..98c8ae988e78 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_incremental.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_incremental.txt @@ -4,35 +4,35 @@ # (I think?) This mimics an interruption to normalization, such that some records were normalized but others were not. # These first records are old data. -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "", "timestamp_col": "", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990900, "data": { "id": "3", "currency": "GBP", "NZD": "3.14", "HKD@spéçiäl & characters": "9.2", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "", "timestamp_col": "", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602637990900, "data": { "id": 3, "currency": "GBP", "NZD": 3.14, "HKD@spéçiäl & characters": 9.2, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} # These records are new data. -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650000000, "data": { "id": "2", "currency": "EUR", "NZD": "3.89", "HKD@spéçiäl & characters": "14.05", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650010000, "data": { "id": "4", "currency": "HKD", "NZD": "1.19", "HKD@spéçiäl & characters": "0.01", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650011000, "data": { "id": "1", "currency": "USD", "date": "2020-10-14", "timestamp_col": "2020-10-14T00:00:00.000-00", "NZD": "1.14", "HKD@spéçiäl & characters": "9.5", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650012000, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "6.39", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650000000, "data": { "id": 2, "currency": "EUR", "NZD": 3.89, "HKD@spéçiäl & characters": 14.05, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650010000, "data": { "id": 4, "currency": "HKD", "NZD": 1.19, "HKD@spéçiäl & characters": 0.01, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650011000, "data": { "id": 1, "currency": "USD", "date": "2020-10-14", "timestamp_col": "2020-10-14T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 9.5, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602650012000, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 6.39, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} # These first records are old data. -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": "2", "currency": "EUR", "date": "", "timestamp_col": "", "NZD": "2.43", "HKD@spéçiäl & characters": "5.4", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990900, "data": { "id": "3", "currency": "GBP", "NZD": "3.14", "HKD@spéçiäl & characters": "9.2", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990800, "data": { "id": 2, "currency": "EUR", "date": "", "timestamp_col": "", "NZD": 2.43, "HKD@spéçiäl & characters": 5.4, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602637990900, "data": { "id": 3, "currency": "GBP", "NZD": 3.14, "HKD@spéçiäl & characters": 9.2, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} # These records are new data. -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650000000, "data": { "id": "2", "currency": "EUR", "NZD": "3.89", "HKD@spéçiäl & characters": "14.05", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650010000, "data": { "id": "4", "currency": "HKD", "NZD": "1.19", "HKD@spéçiäl & characters": "0.01", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650011000, "data": { "id": "1", "currency": "USD", "date": "2020-10-14", "timestamp_col": "2020-10-14T00:00:00.000-00", "NZD": "1.14", "HKD@spéçiäl & characters": "9.5", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650012000, "data": { "id": "5", "currency": "USD", "NZD": "0.01", "HKD@spéçiäl & characters": "6.39", "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650000000, "data": { "id": 2, "currency": "EUR", "NZD": 3.89, "HKD@spéçiäl & characters": 14.05, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650010000, "data": { "id": 4, "currency": "HKD", "NZD": 1.19, "HKD@spéçiäl & characters": 0.01, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650011000, "data": { "id": 1, "currency": "USD", "date": "2020-10-14", "timestamp_col": "2020-10-14T00:00:00.000-00", "NZD": 1.14, "HKD@spéçiäl & characters": 9.5, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602650012000, "data": { "id": 5, "currency": "USD", "NZD": 0.01, "HKD@spéçiäl & characters": 6.39, "HKD_special___characters": "column name collision?", "column`_'with\"_quotes":"ma\"z`d'a"}}} # All of these records are new data. # This record has an _older_ emitted_at than the latest dedup_cdc_excluded record in messages.txt -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"5","name":"vw","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1623849314663","_ab_cdc_lsn":"26975264","_ab_cdc_deleted_at":null},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"5","name":null,"column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1623900000000","_ab_cdc_lsn":"28010252","_ab_cdc_deleted_at":"1623900000000"},"emitted_at":1623900000000}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":5,"name":"vw","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1623849314663,"_ab_cdc_lsn":26975264,"_ab_cdc_deleted_at":null},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":5,"name":null,"column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1623900000000,"_ab_cdc_lsn":28010252,"_ab_cdc_deleted_at":1623900000000},"emitted_at":1623900000000}} # Previously we had a bug where we only respected deletions from the most recent _airbyte_emitted_at. This message tests that ID 5 is still correctly deleted (i.e. marked with _airbyte_active_row = 0). # This record is also deleted in messages_schema_change.txt. -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"8","name":"ford","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1624000000000","_ab_cdc_lsn":"29010252","_ab_cdc_deleted_at":null},"emitted_at":1624000000000}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":8,"name":"ford","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1624000000000,"_ab_cdc_lsn":29010252,"_ab_cdc_deleted_at":null},"emitted_at":1624000000000}} # All of these records are old data. -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"1","name":"mazda","_ab_cdc_updated_at":"1623849130530","_ab_cdc_lsn":"26971624","_ab_cdc_log_pos": "33274","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"2","name":"toyata","_ab_cdc_updated_at":"1623849130549","_ab_cdc_lsn":"26971624","_ab_cdc_log_pos": "33275","_ab_cdc_deleted_at":null},"emitted_at":1623859926}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"2","name":"bmw","_ab_cdc_updated_at":"1623849314535","_ab_cdc_lsn":"26974776","_ab_cdc_log_pos": "33278","_ab_cdc_deleted_at":null},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"3","name":null,"_ab_cdc_updated_at":"1623849314791","_ab_cdc_lsn":"26975440","_ab_cdc_log_pos": "33274","_ab_cdc_deleted_at":"1623849314791"},"emitted_at":1623860160}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"4","name":"lotus","_ab_cdc_updated_at":"1623850868237","_ab_cdc_lsn":"27010048","_ab_cdc_log_pos": "33271","_ab_cdc_deleted_at":null},"emitted_at":1623861660}} -{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":"4","name":null,"_ab_cdc_updated_at":"1623850868371","_ab_cdc_lsn":"27010232","_ab_cdc_log_pos": "33279","_ab_cdc_deleted_at":"1623850868371"},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":1,"name":"mazda","_ab_cdc_updated_at":1623849130530,"_ab_cdc_lsn":26971624,"_ab_cdc_log_pos": 33274,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":2,"name":"toyata","_ab_cdc_updated_at":1623849130549,"_ab_cdc_lsn":26971624,"_ab_cdc_log_pos": 33275,"_ab_cdc_deleted_at":null},"emitted_at":1623859926}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":2,"name":"bmw","_ab_cdc_updated_at":1623849314535,"_ab_cdc_lsn":26974776,"_ab_cdc_log_pos": 33278,"_ab_cdc_deleted_at":null},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":3,"name":null,"_ab_cdc_updated_at":1623849314791,"_ab_cdc_lsn":26975440,"_ab_cdc_log_pos": 33274,"_ab_cdc_deleted_at":1623849314791},"emitted_at":1623860160}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":4,"name":"lotus","_ab_cdc_updated_at":1623850868237,"_ab_cdc_lsn":27010048,"_ab_cdc_log_pos": 33271,"_ab_cdc_deleted_at":null},"emitted_at":1623861660}} +{"type":"RECORD","record":{"stream":"pos_dedup_cdcx","data":{"id":4,"name":null,"_ab_cdc_updated_at":1623850868371,"_ab_cdc_lsn":27010232,"_ab_cdc_log_pos": 33279,"_ab_cdc_deleted_at":1623850868371},"emitted_at":1623861660}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_schema_change.txt b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_schema_change.txt index 0925429d1dde..7190fe88bc35 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_schema_change.txt +++ b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/data_input/messages_schema_change.txt @@ -1,16 +1,16 @@ -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661281900, "data": { "id": "3.14", "currency": "EUR", "new_column": "2.1", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "2.12", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "7"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661291900, "data": { "id": "0.12", "currency": "GBP", "new_column": "3.81", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "3.14", "HKD@spéçiäl & characters": "3.01", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "11"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661381900, "data": { "id": "4.22", "currency": "EUR", "new_column": "89.1", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "3.89", "HKD@spéçiäl & characters": "8.88", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10"}}} -{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661481900, "data": { "id": "1", "currency": "HKD", "new_column": "91.11", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "1.19", "HKD@spéçiäl & characters": "99.1", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10"}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661281900, "data": { "id": 3.14, "currency": "EUR", "new_column": 2.1, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 2.12, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 7}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661291900, "data": { "id": 0.12, "currency": "GBP", "new_column": 3.81, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 3.14, "HKD@spéçiäl & characters": 3.01, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 11}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661381900, "data": { "id": 4.22, "currency": "EUR", "new_column": 89.1, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 3.89, "HKD@spéçiäl & characters": 8.88, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10}}} +{"type": "RECORD", "record": {"stream": "exchange_rate", "emitted_at": 1602661481900, "data": { "id": 1, "currency": "HKD", "new_column": 91.11, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 1.19, "HKD@spéçiäl & characters": 99.1, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661281900, "data": { "id": "3.14", "currency": "EUR", "new_column": "2.1", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "2.43", "HKD@spéçiäl & characters": "2.12", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "7"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661291900, "data": { "id": "0.12", "currency": "GBP", "new_column": "3.81", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "3.14", "HKD@spéçiäl & characters": "3.01", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "11"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661381900, "data": { "id": "4.22", "currency": "EUR", "new_column": "89.1", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "3.89", "HKD@spéçiäl & characters": "8.88", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10"}}} -{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661481900, "data": { "id": "1", "currency": "HKD", "new_column": "91.11", "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": "1.19", "HKD@spéçiäl & characters": "99.1", "column`_'with\"_quotes":"ma\"z`d'a", "USD": "10"}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661281900, "data": { "id": 3.14, "currency": "EUR", "new_column": 2.1, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 2.43, "HKD@spéçiäl & characters": 2.12, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 7}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661291900, "data": { "id": 0.12, "currency": "GBP", "new_column": 3.81, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 3.14, "HKD@spéçiäl & characters": 3.01, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 11}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661381900, "data": { "id": 4.22, "currency": "EUR", "new_column": 89.1, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 3.89, "HKD@spéçiäl & characters": 8.88, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10}}} +{"type": "RECORD", "record": {"stream": "dedup_exchange_rate", "emitted_at": 1602661481900, "data": { "id": 1, "currency": "HKD", "new_column": 91.11, "date": "2020-11-01", "timestamp_col": "2020-11-01T00:00:00Z", "NZD": 1.19, "HKD@spéçiäl & characters": 99.1, "column`_'with\"_quotes":"ma\"z`d'a", "USD": 10}}} -{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":"8","name":"vw","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1623949314663","_ab_cdc_lsn":"26985264","_ab_cdc_deleted_at":null},"emitted_at":1623960160}} -{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":"9","name":"opel","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1623950868109","_ab_cdc_lsn":"28009440","_ab_cdc_deleted_at":null},"emitted_at":1623961660}} -{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":"9","name":null,"column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1623950868371","_ab_cdc_lsn":"28010232","_ab_cdc_deleted_at":"1623950868371"},"emitted_at":1623961660}} +{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":8,"name":"vw","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1623949314663,"_ab_cdc_lsn":26985264,"_ab_cdc_deleted_at":null},"emitted_at":1623960160}} +{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":9,"name":"opel","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1623950868109,"_ab_cdc_lsn":28009440,"_ab_cdc_deleted_at":null},"emitted_at":1623961660}} +{"type":"RECORD","record":{"stream":"renamed_dedup_cdc_excluded","data":{"id":9,"name":null,"column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1623950868371,"_ab_cdc_lsn":28010232,"_ab_cdc_deleted_at":1623950868371},"emitted_at":1623961660}} # This message tests the ability to delete a record which was inserted in a previous sync. See messages_incremental.txt for how it was inserted. -{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":"8","name":"ford","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":"1625000000000","_ab_cdc_lsn":"29020252","_ab_cdc_deleted_at":"1625000000000"},"emitted_at":1625000000000}} +{"type":"RECORD","record":{"stream":"dedup_cdc_excluded","data":{"id":8,"name":"ford","column`_'with\"_quotes":"ma\"z`d'a","_ab_cdc_updated_at":1625000000000,"_ab_cdc_lsn":29020252,"_ab_cdc_deleted_at":1625000000000},"emitted_at":1625000000000}} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_bigquery_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_bigquery_tmp/types_testing_binary_values.sql deleted file mode 100644 index 6b3d183e4961..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_bigquery_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_clickhouse_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_clickhouse_tmp/types_testing_binary_values.sql deleted file mode 100644 index d4596982af56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_clickhouse_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ type_binary() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mssql_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mssql_tmp/types_testing_binary_values.sql deleted file mode 100644 index d4596982af56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mssql_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ type_binary() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mysql_tidb_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mysql_tidb_tmp/types_testing_binary_values.sql deleted file mode 100644 index d4596982af56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_mysql_tidb_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ type_binary() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_oracle_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_oracle_tmp/types_testing_binary_values.sql deleted file mode 100644 index b14ebf3c8137..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_oracle_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) != 'dGVzdA==' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_postgres_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_postgres_tmp/types_testing_binary_values.sql deleted file mode 100644 index d4596982af56..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_postgres_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ type_binary() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_redshift_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_redshift_tmp/types_testing_binary_values.sql deleted file mode 100644 index b14ebf3c8137..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_redshift_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) != 'dGVzdA==' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_snowflake_tmp/types_testing_binary_values.sql b/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_snowflake_tmp/types_testing_binary_values.sql deleted file mode 100644 index a199f033f3c2..000000000000 --- a/airbyte-integrations/bases/base-normalization/integration_tests/resources/test_simple_streams/dbt_test_config/dbt_data_test_binary_tmp/dbt_data_test_snowflake_tmp/types_testing_binary_values.sql +++ /dev/null @@ -1,14 +0,0 @@ -select * from {{ ref('types_testing') }} where -( - id = 1 and ( - cast(property_binary_data as {{ dbt_utils.type_string() }}) != 'test' - ) -) or ( - id = 2 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) or ( - id = 3 and ( - cast(property_binary_data as {{ type_binary() }}) is not null - ) -) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py index 279e9529fa68..f459f5faecd6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_ephemeral.py @@ -188,10 +188,10 @@ def generate_dbt_models(destination_type: DestinationType, test_root_dir: str, c ] } if column_count == 1: - catalog_config["streams"][0]["stream"]["json_schema"]["properties"]["_airbyte_id"] = {"$ref" : "WellKnownTypes.json#/definitions/Integer"} + catalog_config["streams"][0]["stream"]["json_schema"]["properties"]["_airbyte_id"] = {"type": "integer"} else: for column in [dbt_test_utils.random_string(5) for _ in range(column_count)]: - catalog_config["streams"][0]["stream"]["json_schema"]["properties"][column] = {"$ref": "WellKnownTypes.json#/definitions/String"} + catalog_config["streams"][0]["stream"]["json_schema"]["properties"][column] = {"type": "string"} catalog = os.path.join(test_root_dir, "catalog.json") with open(catalog, "w") as fh: fh.write(json.dumps(catalog_config)) diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py index 55f9ca1904a4..0163cd128151 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py +++ b/airbyte-integrations/bases/base-normalization/integration_tests/test_normalization.py @@ -76,8 +76,8 @@ def test_normalization(destination_type: DestinationType, test_resource_name: st if destination_type.value not in dbt_test_utils.get_test_targets(): pytest.skip(f"Destinations {destination_type} is not in NORMALIZATION_TEST_TARGET env variable") if ( - destination_type.value in (DestinationType.ORACLE.value, DestinationType.CLICKHOUSE.value) - and test_resource_name == "test_nested_streams" + destination_type.value in (DestinationType.ORACLE.value, DestinationType.CLICKHOUSE.value) + and test_resource_name == "test_nested_streams" ): pytest.skip(f"Destinations {destination_type} does not support nested streams") @@ -113,7 +113,6 @@ def run_first_normalization(destination_type: DestinationType, test_resource_nam generate_dbt_models(destination_type, test_resource_name, test_root_dir, "models", "catalog.json", dbt_test_utils) # Setup test resources and models setup_dbt_test(destination_type, test_resource_name, test_root_dir) - setup_dbt_binary_test(destination_type, test_resource_name, test_root_dir) dbt_test_utils.dbt_check(destination_type, test_root_dir) # Run dbt process dbt_test_utils.dbt_run(destination_type, test_root_dir, force_full_refresh=True) @@ -219,7 +218,7 @@ def setup_test_dir(destination_type: DestinationType, test_resource_name: str) - def setup_input_raw_data( - destination_type: DestinationType, test_resource_name: str, test_root_dir: str, destination_config: Dict[str, Any] + destination_type: DestinationType, test_resource_name: str, test_root_dir: str, destination_config: Dict[str, Any] ) -> bool: """ We run docker images of destinations to upload test data stored in the messages.txt file for each test case. @@ -302,77 +301,6 @@ def setup_dbt_test(destination_type: DestinationType, test_resource_name: str, t ) -def setup_dbt_binary_test(destination_type: DestinationType, test_resource_name: str, test_root_dir: str): - """ - Prepare the data (copy) for the models for dbt test. - """ - replace_identifiers = os.path.join("resources", test_resource_name, "data_input", "replace_identifiers.json") - - if DestinationType.BIGQUERY == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_bigquery_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.SNOWFLAKE == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_snowflake_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.CLICKHOUSE == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_clickhouse_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.MYSQL == destination_type or DestinationType.TIDB == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_mysql_tidb_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.MSSQL == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_mssql_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.POSTGRES == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_postgres_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.ORACLE == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_oracle_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - if DestinationType.REDSHIFT == destination_type: - copy_test_files( - os.path.join("resources", test_resource_name, "dbt_test_config", "dbt_data_test_binary_tmp/dbt_data_test_redshift_tmp"), - os.path.join(test_root_dir, "models/dbt_data_tests"), - destination_type, - replace_identifiers, - ) - - def setup_dbt_incremental_test(destination_type: DestinationType, test_resource_name: str, test_root_dir: str): """ Prepare the data (copy) for the models for dbt test. @@ -483,6 +411,7 @@ def copy_test_files(src: str, dst: str, destination_type: DestinationType, repla pattern.append(k.replace("\\", r"\\")) replace_value.append(entry[k]) if pattern and replace_value: + def copy_replace_identifiers(src, dst): dbt_test_utils.copy_replace(src, dst, pattern, replace_value) diff --git a/airbyte-integrations/bases/base-normalization/normalization/data_type.py b/airbyte-integrations/bases/base-normalization/normalization/data_type.py deleted file mode 100644 index ccdcd4f9d037..000000000000 --- a/airbyte-integrations/bases/base-normalization/normalization/data_type.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -WELL_KNOWN_TYPE_VAR_NAME = "WellKnownTypes.json" - -REF_TYPE_VAR_NAME = "$ref" -TYPE_VAR_NAME = "type" -ONE_OF_VAR_NAME = "oneOf" - -STRING_TYPE = "WellKnownTypes.json#/definitions/String" -BINARY_DATA_TYPE = "WellKnownTypes.json#/definitions/BinaryData" -BOOLEAN_TYPE = "WellKnownTypes.json#/definitions/Boolean" -INTEGER_TYPE = "WellKnownTypes.json#/definitions/Integer" -NUMBER_TYPE = "WellKnownTypes.json#/definitions/Number" -DATE_TYPE = "WellKnownTypes.json#/definitions/Date" -TIMESTAMP_WITHOUT_TIMEZONE_TYPE = "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" -TIMESTAMP_WITH_TIMEZONE_TYPE = "WellKnownTypes.json#/definitions/TimestampWithTimezone" -TIME_WITH_TIME_ZONE_TYPE = "WellKnownTypes.json#/definitions/TimeWithTimezone" -TIME_WITHOUT_TIME_ZONE_TYPE = "WellKnownTypes.json#/definitions/TimeWithoutTimezone" diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py index 4443d14af843..231588f92903 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/stream_processor.py @@ -2,6 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # + import os import re from enum import Enum @@ -9,7 +10,6 @@ from airbyte_cdk.models.airbyte_protocol import DestinationSyncMode, SyncMode from jinja2 import Template -from normalization import data_type from normalization.destination_type import DestinationType from normalization.transform_catalog import dbt_macro from normalization.transform_catalog.destination_name_transformer import DestinationNameTransformer, transform_json_naming @@ -18,7 +18,6 @@ is_airbyte_column, is_array, is_big_integer, - is_binary_datatype, is_boolean, is_combining_node, is_date, @@ -361,20 +360,12 @@ def find_children_streams(self, from_table: str, column_names: Dict[str, Tuple[s elif is_combining_node(properties[field]): # TODO: merge properties of all combinations pass - elif ( - data_type.TYPE_VAR_NAME not in properties[field] - and data_type.REF_TYPE_VAR_NAME not in properties[field] - and data_type.ONE_OF_VAR_NAME not in properties[field] - ) or (data_type.TYPE_VAR_NAME in properties[field] and is_object(properties[field][data_type.TYPE_VAR_NAME])): + elif "type" not in properties[field] or is_object(properties[field]["type"]): # properties without 'type' field are treated like properties with 'type' = 'object' children_properties = find_properties_object([], field, properties[field]) is_nested_array = False json_column_name = column_names[field][1] - elif ( - data_type.TYPE_VAR_NAME in properties[field] - and is_array(properties[field][data_type.TYPE_VAR_NAME]) - and "items" in properties[field] - ): + elif is_array(properties[field]["type"]) and "items" in properties[field]: quoted_field = column_names[field][1] children_properties = find_properties_object([], field, properties[field]["items"]) is_nested_array = True @@ -465,15 +456,13 @@ def extract_json_column(property_name: str, json_column_name: str, definition: D table_alias = "" json_extract = jinja_call(f"json_extract('{table_alias}', {json_column_name}, {json_path})") - if data_type.REF_TYPE_VAR_NAME in definition or data_type.TYPE_VAR_NAME in definition or data_type.ONE_OF_VAR_NAME in definition: - if data_type.TYPE_VAR_NAME in definition and is_array(definition[data_type.TYPE_VAR_NAME]): + if "type" in definition: + if is_array(definition["type"]): json_extract = jinja_call(f"json_extract_array({json_column_name}, {json_path}, {normalized_json_path})") - if is_simple_property(definition.get("items", {data_type.TYPE_VAR_NAME: "object"})): + if is_simple_property(definition.get("items", {"type": "object"})): json_extract = jinja_call(f"json_extract_string_array({json_column_name}, {json_path}, {normalized_json_path})") - elif data_type.TYPE_VAR_NAME in definition and is_object(definition[data_type.TYPE_VAR_NAME]): + elif is_object(definition["type"]): json_extract = jinja_call(f"json_extract('{table_alias}', {json_column_name}, {json_path}, {normalized_json_path})") - elif data_type.REF_TYPE_VAR_NAME in definition and (is_date(definition) or is_time(definition) or is_datetime(definition)): - json_extract = jinja_call(f"json_extract_scalar({json_column_name}, {json_path}, {normalized_json_path})") elif is_simple_property(definition): json_extract = jinja_call(f"json_extract_scalar({json_column_name}, {json_path}, {normalized_json_path})") @@ -515,44 +504,22 @@ def cast_property_types(self, column_names: Dict[str, Tuple[str, str]]) -> List[ def cast_property_type(self, property_name: str, column_name: str, jinja_column: str) -> Any: # noqa: C901 definition = self.properties[property_name] - if ( - data_type.TYPE_VAR_NAME not in definition - and data_type.REF_TYPE_VAR_NAME not in definition - and data_type.ONE_OF_VAR_NAME not in definition - ): + if "type" not in definition: print(f"WARN: Unknown type for column {property_name} at {self.current_json_path()}") return column_name - elif data_type.TYPE_VAR_NAME in definition and is_array(definition[data_type.TYPE_VAR_NAME]): + elif is_array(definition["type"]): return column_name - elif data_type.TYPE_VAR_NAME in definition and is_object(definition[data_type.TYPE_VAR_NAME]): + elif is_object(definition["type"]): sql_type = jinja_call("type_json()") - # Treat simple types from wider scope TO narrower type: string > boolean > integer > number - elif (data_type.REF_TYPE_VAR_NAME in definition and is_string(definition)) or ( - data_type.ONE_OF_VAR_NAME in definition and is_string(definition) - ): - sql_type = jinja_call("dbt_utils.type_string()") - if self.destination_type == DestinationType.CLICKHOUSE: - trimmed_column_name = f"trim(BOTH '\"' from {column_name})" - sql_type = f"'{sql_type}'" - return f"nullif(accurateCastOrNull({trimmed_column_name}, {sql_type}), 'null') as {column_name}" - elif self.destination_type == DestinationType.MYSQL: - # Cast to `text` datatype. See https://github.com/airbytehq/airbyte/issues/7994 - sql_type = f"{sql_type}(1024)" - - elif (data_type.REF_TYPE_VAR_NAME in definition and is_boolean(definition)) or ( - data_type.ONE_OF_VAR_NAME in definition and is_boolean(definition) - ): + # Treat simple types from narrower to wider scope type: boolean < integer < number < string + elif is_boolean(definition["type"], definition): cast_operation = jinja_call(f"cast_to_boolean({jinja_column})") return f"{cast_operation} as {column_name}" elif is_big_integer(definition): sql_type = jinja_call("type_very_large_integer()") - elif (data_type.REF_TYPE_VAR_NAME in definition and is_long(definition)) or ( - data_type.ONE_OF_VAR_NAME in definition and is_long(definition) - ): + elif is_long(definition["type"], definition): sql_type = jinja_call("dbt_utils.type_bigint()") - elif (data_type.REF_TYPE_VAR_NAME in definition and is_number(definition)) or ( - data_type.ONE_OF_VAR_NAME in definition and is_number(definition) - ): + elif is_number(definition["type"]): sql_type = jinja_call("dbt_utils.type_float()") elif is_datetime(definition): if self.destination_type == DestinationType.SNOWFLAKE: @@ -607,59 +574,21 @@ def cast_property_type(self, property_name: str, column_name: str, jinja_column: return f'nullif(cast({column_name} as {sql_type}), "") as {column_name}' replace_operation = jinja_call(f"empty_string_to_null({jinja_column})") return f"cast({replace_operation} as {sql_type}) as {column_name}" - elif (data_type.REF_TYPE_VAR_NAME in definition and is_binary_datatype(definition)) or ( - data_type.ONE_OF_VAR_NAME in definition and is_binary_datatype(definition) - ): - if self.destination_type.value == DestinationType.POSTGRES.value: - # sql_type = "bytea" - sql_type = jinja_call("type_binary()") - return f"cast(decode({column_name}, 'base64') as {sql_type}) as {column_name}" - elif self.destination_type.value == DestinationType.BIGQUERY.value: - # sql_type = "bytes" - sql_type = jinja_call("type_binary()") - return f"cast(FROM_BASE64({column_name}) as {sql_type}) as {column_name}" - elif self.destination_type.value == DestinationType.MYSQL.value or self.destination_type.value == DestinationType.TIDB.value: - # sql_type = "BINARY" - sql_type = jinja_call("type_binary()") - return f"cast(FROM_BASE64({column_name}) as {sql_type}) as {column_name}" - elif self.destination_type.value == DestinationType.MSSQL.value: - # sql_type = "VARBINARY(MAX)" - sql_type = jinja_call("type_binary()") - return f"CAST({column_name} as XML ).value('.','{sql_type}') as {column_name}" - elif self.destination_type.value == DestinationType.SNOWFLAKE.value: - # sql_type = "VARBINARY" - sql_type = jinja_call("type_binary()") - return f"cast(BASE64_DECODE_BINARY({column_name}) as {sql_type}) as {column_name}" - elif self.destination_type.value == DestinationType.CLICKHOUSE.value: - # sql_type = "VARBINARY" - sql_type = jinja_call("type_binary()") + elif is_string(definition["type"]): + sql_type = jinja_call("dbt_utils.type_string()") + if self.destination_type == DestinationType.CLICKHOUSE: trimmed_column_name = f"trim(BOTH '\"' from {column_name})" - return f"cast(FROM_BASE64({trimmed_column_name}) as {sql_type}) as {column_name}" - else: - sql_type = jinja_call("dbt_utils.type_string()") - + sql_type = f"'{sql_type}'" + return f"nullif(accurateCastOrNull({trimmed_column_name}, {sql_type}), 'null') as {column_name}" + elif self.destination_type == DestinationType.MYSQL: + # Cast to `text` datatype. See https://github.com/airbytehq/airbyte/issues/7994 + sql_type = f"{sql_type}(1024)" else: - if data_type.REF_TYPE_VAR_NAME in definition: - print( - f"WARN: Unknown ref type {definition[data_type.REF_TYPE_VAR_NAME]} for column {property_name} at {self.current_json_path()}" - ) - elif data_type.ONE_OF_VAR_NAME in definition: - print( - f"WARN: Unknown oneOf simple type {definition[data_type.ONE_OF_VAR_NAME]} for column {property_name} at {self.current_json_path()}" - ) - else: - print(f"WARN: Unknown type {definition[data_type.TYPE_VAR_NAME]} for column {property_name} at {self.current_json_path()}") + print(f"WARN: Unknown type {definition['type']} for column {property_name} at {self.current_json_path()}") return column_name if self.destination_type == DestinationType.CLICKHOUSE: - if data_type.REF_TYPE_VAR_NAME in definition and ( - data_type.NUMBER_TYPE in definition[data_type.REF_TYPE_VAR_NAME] - or data_type.INTEGER_TYPE in definition[data_type.REF_TYPE_VAR_NAME] - ): - trimmed_column_name = f"trim(BOTH '\"' from {column_name})" - return f"accurateCastOrNull({trimmed_column_name}, '{sql_type}') as {column_name}" - else: - return f"accurateCastOrNull({column_name}, '{sql_type}') as {column_name}" + return f"accurateCastOrNull({column_name}, '{sql_type}') as {column_name}" else: return f"cast({column_name} as {sql_type}) as {column_name}" @@ -785,17 +714,13 @@ def safe_cast_to_string(definition: Dict, column_name: str, destination_type: De the curly brackets. """ - if ( - data_type.TYPE_VAR_NAME not in definition - and data_type.REF_TYPE_VAR_NAME not in definition - and data_type.ONE_OF_VAR_NAME not in definition - ): + if "type" not in definition: col = column_name - elif data_type.REF_TYPE_VAR_NAME in definition and is_boolean(definition): + elif is_boolean(definition["type"], definition): col = f"boolean_to_string({column_name})" - elif data_type.TYPE_VAR_NAME in definition and is_array(definition[data_type.TYPE_VAR_NAME]): + elif is_array(definition["type"]): col = f"array_to_string({column_name})" - elif data_type.TYPE_VAR_NAME in definition and is_object(definition[data_type.TYPE_VAR_NAME]): + elif is_object(definition["type"]): col = f"object_to_string({column_name})" else: col = column_name @@ -874,7 +799,7 @@ def generate_scd_type_2_model(self, from_table: str, column_names: Dict[str, Tup if ( self.destination_type == DestinationType.BIGQUERY and self.get_cursor_field_property_name(column_names) != self.airbyte_emitted_at - and is_number(self.properties[self.get_cursor_field_property_name(column_names)]) + and is_number(self.properties[self.get_cursor_field_property_name(column_names)]["type"]) ): # partition by float columns is not allowed in BigQuery, cast it to string airbyte_start_at_string = ( @@ -1122,14 +1047,11 @@ def get_primary_key_from_path(self, column_names: Dict[str, Tuple[str, str]], pa if path and len(path) == 1: field = path[0] if not is_airbyte_column(field): - if data_type.REF_TYPE_VAR_NAME in self.properties[field] or data_type.ONE_OF_VAR_NAME in self.properties[field]: - if data_type.ONE_OF_VAR_NAME in self.properties[field]: - property_type = data_type.ONE_OF_VAR_NAME - else: - property_type = data_type.REF_TYPE_VAR_NAME + if "type" in self.properties[field]: + property_type = self.properties[field]["type"] else: property_type = "object" - if is_number(self.properties[field]) or is_object(property_type): + if is_number(property_type) or is_object(property_type): # some destinations don't handle float columns (or complex types) as primary keys, turn them to string return f"cast({column_names[field][0]} as {jinja_call('dbt_utils.type_string()')})" else: @@ -1571,7 +1493,7 @@ def find_properties_object(path: List[str], field: str, properties) -> Dict[str, elif "properties" in properties: # we found a properties object return {current: properties["properties"]} - elif data_type.REF_TYPE_VAR_NAME in properties and is_simple_property(properties): + elif "type" in properties and is_simple_property(properties): # we found a basic type return {current: {}} elif isinstance(properties, dict): diff --git a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py index 10d6ee35af14..862ae722d9ed 100644 --- a/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py +++ b/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/utils.py @@ -2,9 +2,9 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Callable, Set, Union -from normalization import data_type +from typing import Set, Union + from normalization.transform_catalog import dbt_macro @@ -16,77 +16,76 @@ def remove_jinja(command: str) -> str: return str(command).replace("{{ ", "").replace(" }}", "") -def is_type_included(definition: dict, is_type: Callable[[dict], bool]) -> bool: - if data_type.ONE_OF_VAR_NAME in definition: - return bool(any(is_type(option) for option in definition[data_type.ONE_OF_VAR_NAME])) - else: - return is_type(definition) - - -def get_reftype_function(type: str) -> Callable[[dict], bool]: - def is_reftype(definition: dict) -> bool: - return data_type.REF_TYPE_VAR_NAME in definition and type == definition[data_type.REF_TYPE_VAR_NAME] - - return is_reftype - - -def is_string(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.STRING_TYPE)) - - -def is_binary_datatype(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.BINARY_DATA_TYPE)) +def is_string(property_type) -> bool: + return property_type == "string" or "string" in property_type def is_datetime(definition: dict) -> bool: - return is_datetime_with_timezone(definition) or is_datetime_without_timezone(definition) + return ( + is_string(definition["type"]) + and ("format" in definition.keys()) + and (definition["format"] == "date-time" or "date-time" in definition["format"]) + ) def is_datetime_without_timezone(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.TIMESTAMP_WITHOUT_TIMEZONE_TYPE)) + return is_datetime(definition) and definition.get("airbyte_type") == "timestamp_without_timezone" def is_datetime_with_timezone(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.TIMESTAMP_WITH_TIMEZONE_TYPE)) + return is_datetime(definition) and (not definition.get("airbyte_type") or definition.get("airbyte_type") == "timestamp_with_timezone") def is_date(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.DATE_TYPE)) + return ( + is_string(definition["type"]) + and ("format" in definition.keys()) + and (definition["format"] == "date" or "date" in definition["format"]) + ) def is_time(definition: dict) -> bool: - return is_time_with_timezone(definition) or is_time_without_timezone(definition) + return is_string(definition["type"]) and definition.get("format") == "time" def is_time_with_timezone(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.TIME_WITH_TIME_ZONE_TYPE)) + return is_time(definition) and definition.get("airbyte_type") == "time_with_timezone" def is_time_without_timezone(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.TIME_WITHOUT_TIME_ZONE_TYPE)) + return is_time(definition) and definition.get("airbyte_type") == "time_without_timezone" -def is_number(definition: dict) -> bool: - if is_string(definition): +def is_number(property_type) -> bool: + if is_string(property_type): # Handle union type, give priority to wider scope types return False - return is_type_included(definition, get_reftype_function(data_type.NUMBER_TYPE)) + return property_type == "number" or "number" in property_type -# this is obsolete type that will not be used in new datatypes def is_big_integer(definition: dict) -> bool: - return False + return "airbyte_type" in definition and definition["airbyte_type"] == "big_integer" -def is_long(definition: dict) -> bool: - return is_type_included(definition, get_reftype_function(data_type.INTEGER_TYPE)) +def is_long(property_type, definition: dict) -> bool: + # Check specifically for {type: number, airbyte_type: integer} + if ( + (property_type == "number" or "number" in property_type) + and "airbyte_type" in definition + and definition["airbyte_type"] == "integer" + ): + return True + if is_string(property_type) or is_number(property_type): + # Handle union type, give priority to wider scope types + return False + return property_type == "integer" or "integer" in property_type -def is_boolean(definition: dict) -> bool: - if is_string(definition) or is_number(definition) or is_big_integer(definition) or is_long(definition): +def is_boolean(property_type, definition: dict) -> bool: + if is_string(property_type) or is_number(property_type) or is_big_integer(definition) or is_long(property_type, definition): # Handle union type, give priority to wider scope types return False - return is_type_included(definition, get_reftype_function(data_type.BOOLEAN_TYPE)) + return property_type == "boolean" or "boolean" in property_type def is_array(property_type) -> bool: @@ -102,24 +101,18 @@ def is_airbyte_column(name: str) -> bool: def is_simple_property(definition: dict) -> bool: + if "type" not in definition: + property_type = "object" + else: + property_type = definition["type"] return ( - is_string(definition) + is_string(property_type) or is_big_integer(definition) - or is_long(definition) - or is_number(definition) - or is_boolean(definition) - or is_date(definition) - or is_time(definition) - or is_datetime(definition) - or is_binary_datatype(definition) + or is_long(property_type, definition) + or is_number(property_type) + or is_boolean(property_type, definition) ) def is_combining_node(properties: dict) -> Set[str]: - # this case appears when we have analog of old protocol like id: {type:[number, string]} and it's handled separately - if data_type.ONE_OF_VAR_NAME in properties and any( - data_type.WELL_KNOWN_TYPE_VAR_NAME in option[data_type.REF_TYPE_VAR_NAME] for option in properties[data_type.ONE_OF_VAR_NAME] - ): - return set() - else: - return set(properties).intersection({"anyOf", "oneOf", "allOf"}) + return set(properties).intersection({"anyOf", "oneOf", "allOf"}) diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog.json index 8b1bb673bd45..7ffa2f36d442 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog.json +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/long_name_truncate_collisions_catalog.json @@ -7,7 +7,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -27,7 +27,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -46,7 +46,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog.json index 882af94c7d22..94e6b4a798d9 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog.json +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/nested_catalog.json @@ -7,19 +7,19 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "body": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "title": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "status": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "adlabels": { "type": ["null", "array"], @@ -27,22 +27,24 @@ "type": "object", "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "created_time": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + "type": "string", + "format": "date-time" }, "updated_time": { - "$ref": "WellKnownTypes.json#/definitions/TimestampWithoutTimezone" + "type": "string", + "format": "date-time" } } } }, "link_url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "image_crops": { "type": ["null", "object"], @@ -52,7 +54,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -61,7 +63,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -70,7 +72,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -79,7 +81,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -88,7 +90,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -97,7 +99,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -106,7 +108,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } } @@ -116,22 +118,22 @@ "type": ["null", "object"], "properties": { "page_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "link_data": { "type": ["null", "object"], "properties": { "link": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "caption": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "message": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "image_crops": { "type": ["null", "object"], @@ -141,7 +143,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -150,7 +152,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -159,7 +161,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -168,7 +170,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -177,7 +179,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -186,7 +188,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } }, @@ -195,7 +197,7 @@ "items": { "type": ["null", "array"], "items": { - "$ref": "WellKnownTypes.json#/definitions/Integer" + "type": ["null", "integer"] } } } @@ -210,13 +212,13 @@ "type": "object", "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } } @@ -227,13 +229,13 @@ "type": "object", "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } } @@ -244,13 +246,13 @@ "type": "object", "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } } @@ -261,16 +263,16 @@ "type": "object", "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "class": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "package": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } } @@ -282,7 +284,7 @@ "type": ["null", "object"], "properties": { "message": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -290,27 +292,27 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "caption": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "image_hash": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "page_welcome_message": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "branded_content_sponsor_page_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "branded_content_sponsor_relationship": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, "instagram_actor_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -321,13 +323,13 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -335,10 +337,10 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "should_fallback": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -346,13 +348,13 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -360,7 +362,7 @@ "type": ["null", "object"], "properties": { "app_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -368,13 +370,13 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_store_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -382,13 +384,13 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "package": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } }, @@ -396,13 +398,13 @@ "type": ["null", "object"], "properties": { "url": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" }, "app_name": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": "string" } } } diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog.json b/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog.json index 3197a208302c..336cf17d7194 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog.json +++ b/airbyte-integrations/bases/base-normalization/unit_tests/resources/un-nesting_collisions_catalog.json @@ -8,7 +8,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -28,13 +28,13 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] }, "stream_name": { "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } } @@ -56,7 +56,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, @@ -76,7 +76,7 @@ "type": ["null", "object"], "properties": { "id": { - "$ref": "WellKnownTypes.json#/definitions/String" + "type": ["null", "string"] } } }, diff --git a/airbyte-integrations/bases/base-normalization/unit_tests/test_stream_processor.py b/airbyte-integrations/bases/base-normalization/unit_tests/test_stream_processor.py index 232fe5652b63..cfc48bacbef1 100644 --- a/airbyte-integrations/bases/base-normalization/unit_tests/test_stream_processor.py +++ b/airbyte-integrations/bases/base-normalization/unit_tests/test_stream_processor.py @@ -64,13 +64,13 @@ def test_cursor_field(cursor_field: List[str], expecting_exception: bool, expect @pytest.mark.parametrize( "primary_key, column_type, expecting_exception, expected_primary_keys, expected_final_primary_key_string", [ - ([["id"]], "WellKnownTypes.json#/definitions/String", False, ["id"], "{{ adapter.quote('id') }}"), - ([["id"]], "WellKnownTypes.json#/definitions/Number", False, ["id"], "cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }})"), - ([["first_name"], ["last_name"]], "WellKnownTypes.json#/definitions/String", False, ["first_name", "last_name"], "first_name, last_name"), - ([["float_id"]], "WellKnownTypes.json#/definitions/Number", False, ["float_id"], "cast(float_id as {{ dbt_utils.type_string() }})"), - ([["_airbyte_emitted_at"]], "WellKnownTypes.json#/definitions/String", False, [], "cast(_airbyte_emitted_at as {{ dbt_utils.type_string() }})"), - (None, "WellKnownTypes.json#/definitions/String", True, [], ""), - ([["parent", "nested_field"]], "WellKnownTypes.json#/definitions/String", True, [], ""), + ([["id"]], "string", False, ["id"], "{{ adapter.quote('id') }}"), + ([["id"]], "number", False, ["id"], "cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }})"), + ([["first_name"], ["last_name"]], "string", False, ["first_name", "last_name"], "first_name, last_name"), + ([["float_id"]], "number", False, ["float_id"], "cast(float_id as {{ dbt_utils.type_string() }})"), + ([["_airbyte_emitted_at"]], "string", False, [], "cast(_airbyte_emitted_at as {{ dbt_utils.type_string() }})"), + (None, "string", True, [], ""), + ([["parent", "nested_field"]], "string", True, [], ""), ], ) def test_primary_key( @@ -91,7 +91,7 @@ def test_primary_key( cursor_field=[], primary_key=primary_key, json_column_name="json_column_name", - properties={key: {"$ref": column_type} for key in expected_primary_keys}, + properties={key: {"type": column_type} for key in expected_primary_keys}, tables_registry=TableNameRegistry(DestinationType.POSTGRES), from_table="", ) From b7af3a858412dfd822fada90c203814b6327a7cf Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 16:58:09 -0800 Subject: [PATCH 4/7] revert dest definitions --- .../seed/destination_definitions.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index 49044b747ff7..002404443f58 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -45,7 +45,7 @@ icon: bigquery.svg normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: bigquery supportsDbt: true resourceRequirements: @@ -91,7 +91,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-clickhouse - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: clickhouse supportsDbt: true - name: Cloudflare R2 @@ -213,7 +213,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mssql - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: mssql supportsDbt: true - name: MeiliSearch @@ -239,7 +239,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-mysql - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: mysql supportsDbt: true - name: Oracle @@ -251,7 +251,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-oracle - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: oracle supportsDbt: true - name: Postgres @@ -263,7 +263,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: postgres supportsDbt: true - name: Pulsar @@ -295,7 +295,7 @@ icon: redshift.svg normalizationConfig: normalizationRepository: airbyte/normalization-redshift - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: redshift supportsDbt: true resourceRequirements: @@ -353,7 +353,7 @@ icon: snowflake.svg normalizationConfig: normalizationRepository: airbyte/normalization-snowflake - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: snowflake supportsDbt: true resourceRequirements: @@ -407,7 +407,7 @@ releaseStage: alpha normalizationConfig: normalizationRepository: airbyte/normalization-tidb - normalizationTag: 0.3.0 + normalizationTag: 0.2.25 normalizationIntegrationType: tidb supportsDbt: true - name: Typesense From 4abde74b6166c5c10926dd260db6358b7939c5c1 Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 17:02:47 -0800 Subject: [PATCH 5/7] also dockerfile --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index eb985d00e88d..71cbb2f88b6e 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,5 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] -LABEL io.airbyte.version=0.3.0 +LABEL io.airbyte.version=0.2.25 LABEL io.airbyte.name=airbyte/normalization From f3ae4ae153fc7fc236a665a9269b8de1851775db Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 19:34:58 -0800 Subject: [PATCH 6/7] re-add to changelog --- docs/understanding-airbyte/basic-normalization.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/understanding-airbyte/basic-normalization.md b/docs/understanding-airbyte/basic-normalization.md index 876b14edded2..1fe36f11de62 100644 --- a/docs/understanding-airbyte/basic-normalization.md +++ b/docs/understanding-airbyte/basic-normalization.md @@ -353,8 +353,10 @@ Therefore, in order to "upgrade" to the desired normalization version, you need | Airbyte Version | Normalization Version | Date | Pull Request | Subject | |:----------------|:----------------------|:-----------|:-------------------------------------------------------------------------------------------------------------------------------------|:---------------------------------------------------------------------------------------------------------------------| -| | 0.3.0 | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721) | Update normalization to airbyte-protocol v1.0.0 | -| | 0.2.25 | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update Clickhouse dbt version | +| | 0.3.2 (broken, do not use) | 2023-01-31 | [\#22165](https://github.com/airbytehq/airbyte/pull/22165) | Fix support for non-object top-level schemas | +| | 0.3.1 (broken, do not use) | 2023-01-31 | [\#22161](https://github.com/airbytehq/airbyte/pull/22161) | Fix handling for combined primitive types | +| | 0.3.0 (broken, do not use) | 2023-01-30 | [\#19721](https://github.com/airbytehq/airbyte/pull/19721) | Update normalization to airbyte-protocol v1.0.0 | +| | 0.2.25 | 2022-12-05 | [\#19573](https://github.com/airbytehq/airbyte/pull/19573) | Update Clickhouse dbt version | | | 0.2.24 | 2022-11-01 | [\#18015](https://github.com/airbytehq/airbyte/pull/18015) | Add a drop table hook that drops *_scd tables after overwrite/reset | | | 0.2.23 | 2022-10-12 | [\#17483](https://github.com/airbytehq/airbyte/pull/17483) (published in [\#17896](https://github.com/airbytehq/airbyte/pull/17896)) | Remove unnecessary `Native Port` config option | | | 0.2.22 | 2022-09-05 | [\#16339](https://github.com/airbytehq/airbyte/pull/16339) | Update Clickhouse DBT to 1.1.8 | From 866358b011a1760e4329f3da39cae25ac7b2437f Mon Sep 17 00:00:00 2001 From: Edward Gao Date: Wed, 1 Feb 2023 19:35:45 -0800 Subject: [PATCH 7/7] add comment in dockerfile --- airbyte-integrations/bases/base-normalization/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte-integrations/bases/base-normalization/Dockerfile b/airbyte-integrations/bases/base-normalization/Dockerfile index 71cbb2f88b6e..1bb67ccc67f2 100644 --- a/airbyte-integrations/bases/base-normalization/Dockerfile +++ b/airbyte-integrations/bases/base-normalization/Dockerfile @@ -28,5 +28,7 @@ WORKDIR /airbyte ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh" ENTRYPOINT ["/airbyte/entrypoint.sh"] +# 0.3.x is tombstoned. +# The next minor bump should go directly to 0.4.0 LABEL io.airbyte.version=0.2.25 LABEL io.airbyte.name=airbyte/normalization