From 0723de927ad3bd560b5954b415caf2026c548563 Mon Sep 17 00:00:00 2001 From: Vinoth Govindarajan Date: Wed, 17 Nov 2021 15:27:11 -0800 Subject: [PATCH] Commented out the Hudi integration test until we have the hudi 0.10.0 version --- .../test_incremental_strategies.py | 38 +++++++++---------- tests/integration/spark-thrift-hudi.dbtspec | 34 ----------------- 2 files changed, 19 insertions(+), 53 deletions(-) delete mode 100644 tests/integration/spark-thrift-hudi.dbtspec diff --git a/tests/integration/incremental_strategies/test_incremental_strategies.py b/tests/integration/incremental_strategies/test_incremental_strategies.py index 1fc217cbc..839f167e6 100644 --- a/tests/integration/incremental_strategies/test_incremental_strategies.py +++ b/tests/integration/incremental_strategies/test_incremental_strategies.py @@ -81,25 +81,25 @@ def run_and_test(self): def test_delta_strategies_databricks_cluster(self): self.run_and_test() - -class TestHudiStrategies(TestIncrementalStrategies): - @property - def models(self): - return "models_hudi" - - def run_and_test(self): - self.seed_and_run_once() - self.assertTablesEqual("append", "expected_append") - self.assertTablesEqual("merge_no_key", "expected_append") - self.assertTablesEqual("merge_unique_key", "expected_upsert") - self.assertTablesEqual( - "insert_overwrite_no_partitions", "expected_overwrite") - self.assertTablesEqual( - "insert_overwrite_partitions", "expected_upsert") - - @use_profile("apache_spark") - def test_hudi_strategies_apache_spark(self): - self.run_and_test() +# Uncomment this hudi integration test after the hudi 0.10.0 release to make it work. +# class TestHudiStrategies(TestIncrementalStrategies): +# @property +# def models(self): +# return "models_hudi" +# +# def run_and_test(self): +# self.seed_and_run_once() +# self.assertTablesEqual("append", "expected_append") +# self.assertTablesEqual("merge_no_key", "expected_append") +# self.assertTablesEqual("merge_unique_key", "expected_upsert") +# self.assertTablesEqual( +# "insert_overwrite_no_partitions", "expected_overwrite") +# self.assertTablesEqual( +# "insert_overwrite_partitions", "expected_upsert") +# +# @use_profile("apache_spark") +# def test_hudi_strategies_apache_spark(self): +# self.run_and_test() class TestBadStrategies(TestIncrementalStrategies): diff --git a/tests/integration/spark-thrift-hudi.dbtspec b/tests/integration/spark-thrift-hudi.dbtspec deleted file mode 100644 index cee52a9e8..000000000 --- a/tests/integration/spark-thrift-hudi.dbtspec +++ /dev/null @@ -1,34 +0,0 @@ -target: - type: spark - host: localhost - user: dbt - method: thrift - port: 10000 - connect_retries: 5 - connect_timeout: 60 - schema: "analytics_{{ var('_dbt_random_suffix') }}" -projects: - - overrides: snapshot_strategy_check_cols - dbt_project_yml: &file_format_hudi - # we're going to UPDATE the seed tables as part of testing, so we must make them hudi format - seeds: - dbt_test_project: - file_format: hudi - snapshots: - dbt_test_project: - file_format: hudi - - overrides: snapshot_strategy_timestamp - dbt_project_yml: *file_format_hudi -sequences: - test_dbt_empty: empty - test_dbt_base: base - test_dbt_ephemeral: ephemeral - # snapshots require hudi format - test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp - test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols - test_dbt_data_test: data_test - test_dbt_schema_test: schema_test - # the local cluster currently tests on spark 2.x, which does not support this - # if we upgrade it to 3.x, we can enable this test - # test_dbt_incremental: incremental - # test_dbt_ephemeral_data_tests: data_test_ephemeral_models