Un-revert add /tmp emptyDir volume to connector pods (#11511)

* Revert "Revert "add /tmp emptyDir volume to connector pods (#10761)" (#11053)"

This reverts commit eea5156.

* prettier

* bump version of base-normalization to pick up /tmp -> /dbt-tmp change

* change /dbt-tmp/dbt_modules to /dbt

* Regenerate test output files

* add to changelog

Co-authored-by: Christophe Duong <christophe.duong@gmail.com>
Co-authored-by: Edward Gao <edward.gao@airbyte.io>

airbyte-integrations/bases/base-normalization/Dockerfile

@@ -28,5 +28,5 @@ WORKDIR /airbyte
 ENV AIRBYTE_ENTRYPOINT "/airbyte/entrypoint.sh"
 ENTRYPOINT ["/airbyte/entrypoint.sh"]
 
-LABEL io.airbyte.version=0.1.74
+LABEL io.airbyte.version=0.1.75
 LABEL io.airbyte.name=airbyte/normalization

airbyte-integrations/bases/base-normalization/dbt-project-template-clickhouse/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/dbt-project-template-mssql/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/dbt-project-template-mysql/dbt_project.yml

@@ -4,13 +4,13 @@
 # Name your package! Package names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'airbyte_utils'
-version: '1.0'
+name: "airbyte_utils"
+version: "1.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project. Profiles contain
 # database connection information, and should be configured in the  ~/.dbt/profiles.yml file
-profile: 'normalize'
+profile: "normalize"
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that source models can be found
@@ -22,18 +22,18 @@ test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "../build"  # directory which will store compiled SQL files
-log-path: "../logs"  # directory which will store DBT logs
-modules-path: "/tmp/dbt_modules"  # directory which will store external DBT dependencies
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+modules-path: "/dbt" # directory which will store external DBT dependencies
 
-clean-targets:         # directories to be removed by `dbt clean`
-    - "build"
-    - "dbt_modules"
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
 
 quoting:
   database: true
-# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
-# all schemas should be unquoted
+  # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
+  # all schemas should be unquoted
   schema: false
   identifier: true
 
@@ -60,4 +60,4 @@ models:
         +materialized: view
 
 vars:
-  dbt_utils_dispatch_list: ['airbyte_utils']
+  dbt_utils_dispatch_list: ["airbyte_utils"]

airbyte-integrations/bases/base-normalization/dbt-project-template-oracle/dbt_project.yml

@@ -4,13 +4,13 @@
 # Name your package! Package names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'airbyte_utils'
-version: '1.0'
+name: "airbyte_utils"
+version: "1.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project. Profiles contain
 # database connection information, and should be configured in the  ~/.dbt/profiles.yml file
-profile: 'normalize'
+profile: "normalize"
 
 # These configurations specify where dbt should look for different types of files.
 # The `source-paths` config, for example, states that source models can be found
@@ -22,13 +22,13 @@ test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "../build"  # directory which will store compiled SQL files
-log-path: "../logs"  # directory which will store DBT logs
-modules-path: "/tmp/dbt_modules"  # directory which will store external DBT dependencies
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+modules-path: "/dbt" # directory which will store external DBT dependencies
 
-clean-targets:         # directories to be removed by `dbt clean`
-    - "build"
-    - "dbt_modules"
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
 
 quoting:
   database: false
@@ -58,4 +58,4 @@ models:
         +materialized: view
 
 vars:
-  dbt_utils_dispatch_list: ['airbyte_utils']
+  dbt_utils_dispatch_list: ["airbyte_utils"]

airbyte-integrations/bases/base-normalization/dbt-project-template-snowflake/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/dbt-project-template/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/integration_tests/dbt_integration_test.py

@@ -420,7 +420,7 @@ def run_check_dbt_command(normalization_image: str, command: str, cwd: str, forc
                 "-v",
                 f"{cwd}/logs:/logs",
                 "-v",
-                "/tmp:/tmp",
+                f"{cwd}/build/dbt_packages:/dbt",
                 "--network",
                 "host",
                 "--entrypoint",

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_nested_streams/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml

@@ -24,7 +24,7 @@ macro-paths: ["macros"]
 
 target-path: "../build" # directory which will store compiled SQL files
 log-path: "../logs" # directory which will store DBT logs
-packages-install-path: "/tmp/dbt_modules" # directory which will store external DBT dependencies
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
 clean-targets: # directories to be removed by `dbt clean`
   - "build"

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml

@@ -4,40 +4,40 @@
 # Name your package! Package names should contain only lowercase characters
 # and underscores. A good package name should reflect your organization's
 # name or the intended use of these models
-name: 'airbyte_utils'
-version: '1.0'
+name: "airbyte_utils"
+version: "1.0"
 config-version: 2
 
 # This setting configures which "profile" dbt uses for this project. Profiles contain
 # database connection information, and should be configured in the  ~/.dbt/profiles.yml file
-profile: 'normalize'
+profile: "normalize"
 
 # These configurations specify where dbt should look for different types of files.
-# The `source-paths` config, for example, states that source models can be found
+# The `model-paths` config, for example, states that source models can be found
 # in the "models/" directory. You probably won't need to change these!
-source-paths: ["models"]
+model-paths: ["models"]
 docs-paths: ["docs"]
 analysis-paths: ["analysis"]
 test-paths: ["tests"]
-data-paths: ["data"]
+seed-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "../build"  # directory which will store compiled SQL files
-log-path: "../logs"  # directory which will store DBT logs
-modules-path: "/tmp/dbt_modules"  # directory which will store external DBT dependencies
+target-path: "../build" # directory which will store compiled SQL files
+log-path: "../logs" # directory which will store DBT logs
+packages-install-path: "/dbt" # directory which will store external DBT dependencies
 
-clean-targets:         # directories to be removed by `dbt clean`
-    - "build"
-    - "dbt_modules"
+clean-targets: # directories to be removed by `dbt clean`
+  - "build"
+  - "dbt_modules"
 
 quoting:
   database: true
-# Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
-# all schemas should be unquoted
+  # Temporarily disabling the behavior of the ExtendedNameTransformer on table/schema names, see (issue #1785)
+  # all schemas should be unquoted
   schema: false
   identifier: true
 
-# You can define configurations for models in the `source-paths` directory here.
+# You can define configurations for models in the `model-paths` directory here.
 # Using these configurations, you can enable or disable models, change how they
 # are materialized, and more!
 models:
@@ -61,4 +61,4 @@ models:
 
 dispatch:
   - macro_namespace: dbt_utils
-    search_order: ['airbyte_utils', 'dbt_utils']
+    search_order: ["airbyte_utils", "dbt_utils"]

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql

@@ -16,15 +16,15 @@ with __dbt__cte__exchange_rate_ab1 as (
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 -- depends_on: test_normalization._airbyte_raw_exchange_rate
 select
-    JSONExtractRaw(_airbyte_data, 'id') as id,
-    JSONExtractRaw(_airbyte_data, 'currency') as currency,
-    JSONExtractRaw(_airbyte_data, 'date') as date,
-    JSONExtractRaw(_airbyte_data, 'timestamp_col') as timestamp_col,
-    JSONExtractRaw(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
-    JSONExtractRaw(_airbyte_data, 'HKD_special___characters') as HKD_special___characters,
-    JSONExtractRaw(_airbyte_data, 'NZD') as NZD,
-    JSONExtractRaw(_airbyte_data, 'USD') as USD,
-    JSONExtractRaw(_airbyte_data, 'column`_''with\"_quotes') as "column`_'with""_quotes",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'id') as id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'currency') as currency,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'date') as date,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'timestamp_col') as timestamp_col,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD_special___characters') as HKD_special___characters,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'NZD') as NZD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'USD') as USD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'column`_''with\"_quotes') as "column`_'with""_quotes",
     _airbyte_ab_id,
     _airbyte_emitted_at,
     now() as _airbyte_normalized_at

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql

@@ -9,14 +9,14 @@ with __dbt__cte__dedup_exchange_rate_ab1 as (
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate
 select
-    JSONExtractRaw(_airbyte_data, 'id') as id,
-    JSONExtractRaw(_airbyte_data, 'currency') as currency,
-    JSONExtractRaw(_airbyte_data, 'date') as date,
-    JSONExtractRaw(_airbyte_data, 'timestamp_col') as timestamp_col,
-    JSONExtractRaw(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
-    JSONExtractRaw(_airbyte_data, 'HKD_special___characters') as HKD_special___characters,
-    JSONExtractRaw(_airbyte_data, 'NZD') as NZD,
-    JSONExtractRaw(_airbyte_data, 'USD') as USD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'id') as id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'currency') as currency,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'date') as date,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'timestamp_col') as timestamp_col,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD_special___characters') as HKD_special___characters,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'NZD') as NZD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'USD') as USD,
     _airbyte_ab_id,
     _airbyte_emitted_at,
     now() as _airbyte_normalized_at

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql

@@ -9,13 +9,13 @@ with __dbt__cte__multiple_column_names_conflicts_ab1 as (
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 -- depends_on: test_normalization._airbyte_raw_multiple_column_names_conflicts
 select
-    JSONExtractRaw(_airbyte_data, 'id') as id,
-    JSONExtractRaw(_airbyte_data, 'User Id') as "User Id",
-    JSONExtractRaw(_airbyte_data, 'user_id') as user_id,
-    JSONExtractRaw(_airbyte_data, 'User id') as "User id",
-    JSONExtractRaw(_airbyte_data, 'user id') as "user id",
-    JSONExtractRaw(_airbyte_data, 'User@Id') as "User@Id",
-    JSONExtractRaw(_airbyte_data, 'UserId') as UserId,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'id') as id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'User Id') as "User Id",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'user_id') as user_id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'User id') as "User id",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'user id') as "user id",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'User@Id') as "User@Id",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'UserId') as UserId,
     _airbyte_ab_id,
     _airbyte_emitted_at,
     now() as _airbyte_normalized_at

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql

@@ -16,15 +16,15 @@ with __dbt__cte__exchange_rate_ab1 as (
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 -- depends_on: test_normalization._airbyte_raw_exchange_rate
 select
-    JSONExtractRaw(_airbyte_data, 'id') as id,
-    JSONExtractRaw(_airbyte_data, 'currency') as currency,
-    JSONExtractRaw(_airbyte_data, 'date') as date,
-    JSONExtractRaw(_airbyte_data, 'timestamp_col') as timestamp_col,
-    JSONExtractRaw(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
-    JSONExtractRaw(_airbyte_data, 'HKD_special___characters') as HKD_special___characters,
-    JSONExtractRaw(_airbyte_data, 'NZD') as NZD,
-    JSONExtractRaw(_airbyte_data, 'USD') as USD,
-    JSONExtractRaw(_airbyte_data, 'column`_''with\"_quotes') as "column`_'with""_quotes",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'id') as id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'currency') as currency,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'date') as date,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'timestamp_col') as timestamp_col,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD_special___characters') as HKD_special___characters,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'NZD') as NZD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'USD') as USD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'column`_''with\"_quotes') as "column`_'with""_quotes",
     _airbyte_ab_id,
     _airbyte_emitted_at,
     now() as _airbyte_normalized_at

airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql

@@ -9,14 +9,14 @@ with __dbt__cte__dedup_exchange_rate_ab1 as (
 -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema
 -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate
 select
-    JSONExtractRaw(_airbyte_data, 'id') as id,
-    JSONExtractRaw(_airbyte_data, 'currency') as currency,
-    JSONExtractRaw(_airbyte_data, 'date') as date,
-    JSONExtractRaw(_airbyte_data, 'timestamp_col') as timestamp_col,
-    JSONExtractRaw(_airbyte_data, 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
-    JSONExtractRaw(_airbyte_data, 'HKD_special___characters') as HKD_special___characters,
-    JSONExtractRaw(_airbyte_data, 'NZD') as NZD,
-    JSONExtractRaw(_airbyte_data, 'USD') as USD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'id') as id,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'currency') as currency,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'date') as date,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'timestamp_col') as timestamp_col,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD@spéçiäl & characters') as "HKD@spéçiäl & characters",
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'HKD_special___characters') as HKD_special___characters,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'NZD') as NZD,
+    JSONExtractRaw(assumeNotNull(_airbyte_data), 'USD') as USD,
     _airbyte_ab_id,
     _airbyte_emitted_at,
     now() as _airbyte_normalized_at