Tests: Unlock more test cases by removing ignores, skip only selectively

.github/workflows/integration-tests.yml

@@ -92,7 +92,7 @@ jobs:
           - 4200:4200
           - 5432:5432
         env:
-          CRATE_HEAP_SIZE: 4g
+          CRATE_HEAP_SIZE: 8g
 
     steps:
       - name: Check out repository
@@ -129,6 +129,11 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Configure CrateDB
+        run: |
+          pip install crash
+          crash -c "SET GLOBAL PERSISTENT cluster.max_shards_per_node = 5000;"
+
       - name: Run integration tests
         run: hatch run integration-tests
         env:

DEVELOP.md

@@ -0,0 +1,23 @@
+# dbt-cratedb2 sandbox
+
+## Setup
+```shell
+uv venv
+source .venv/bin/activate
+uv pip install hatch
+```
+
+## Usage
+Start CrateDB.
+```shell
+docker run --rm -it --name=cratedb \
+  --publish=4200:4200 --publish=5432:5432 \
+  --env=CRATE_HEAP_SIZE=8g crate/crate:nightly \
+  -Ccluster.max_shards_per_node=5000 \
+  -Cdiscovery.type=single-node
+```
+Invoke software tests.
+```shell
+hatch run unit-tests
+hatch run integration-tests
+```

dbt/adapters/cratedb/connections.py

@@ -46,6 +46,8 @@ def commit(self):
     @contextmanager
     def exception_handler(self, sql):
         try:
+            # CrateDB: This is a good spot for tracing SQL statements.
+            # print("SQL:", sql)
             yield
             # CrateDB needs write synchronization after DML operations.
             # TODO: Only enable optionally?

dbt/adapters/cratedb/impl.py

@@ -127,7 +127,7 @@ def get_rows_different_sql(
         columns: the number of rows that are different between the two
         relations and the number of mismatched rows.
 
-        FIXME: CrateDB does not support the EXCEPT operation.
+        FIXME: CrateDB: `EXCEPT` not supported.
         """
         # This method only really exists for test reasons.
         return COLUMNS_EQUAL_SQL
@@ -139,6 +139,8 @@ def run_sql_for_tests(self, sql, fetch, conn):
         stmt = SQLStatement(sql)
         cursor = conn.handle.cursor()
         try:
+            # CrateDB: This is a good spot for tracing SQL statements.
+            # print("SQL:", sql)
             cursor.execute(sql)
             if stmt.is_dml:
                 for table in stmt.tables:

dbt/adapters/cratedb/relation_configs/constants.py

@@ -1 +1 @@
-MAX_CHARACTERS_IN_IDENTIFIER = 128
+MAX_CHARACTERS_IN_IDENTIFIER = 256

dbt/include/cratedb/macros/adapters.sql

@@ -11,7 +11,7 @@
 {% endmacro %}
 
 
-{# Needs an override because CrateDB does not support `CREATE TEMPORARY TABLE` #}
+{# Needs an override because CrateDB lacks support for `CREATE TEMPORARY TABLE` #}
 {% macro cratedb__create_table_as(temporary, relation, sql) -%}
   {%- set unlogged = config.get('unlogged', default=false) -%}
   {%- set sql_header = config.get('sql_header', none) -%}
@@ -61,7 +61,7 @@
 {% endmacro %}
 
 
-{# CrateDB: Does not support `COMMENT ON` statements. #}
+{# CrateDB: `COMMENT ON` not supported. #}
 {% macro cratedb__alter_relation_comment(relation, comment) %}
   {% set escaped_comment = postgres_escape_comment(comment) %}
   {% if relation.type == 'materialized_view' -%}
@@ -74,7 +74,7 @@
 {% endmacro %}
 
 
-{# CrateDB: Does not support `COMMENT ON` statements. #}
+{# CrateDB: `COMMENT ON` not supported. #}
 {% macro cratedb__alter_column_comment(relation, column_dict) %}
   {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute="name") | list %}
   {% for column_name in column_dict if (column_name in existing_columns) %}

dbt/include/cratedb/macros/catalog.sql

@@ -6,7 +6,7 @@
       If the user has multiple databases set and the first one is wrong, this will fail.
       But we won't fail in the case where there are multiple quoting-difference-only dbs, which is better.
 
-      FIXME: CrateDB does not have `pg_is_other_temp_schema`.
+      FIXME: CrateDB: Missing function `pg_is_other_temp_schema`.
     #}
     {% set database = information_schema.database %}
     {{ adapter.verify_database(database) }}
@@ -43,7 +43,7 @@
         {%- if not loop.last %} or {% endif -%}
       {%- endfor -%}
     )
-      -- FIXME: Do not use `pg_is_other_temp_schema`, CrateDB does not have it.
+      -- FIXME: Do not use `pg_is_other_temp_schema`, not provided by CrateDB.
       -- and not pg_is_other_temp_schema(sch.oid) -- not a temporary schema belonging to another session
       and tbl.relpersistence in ('p', 'u') -- [p]ermanent table or [u]nlogged table. Exclude [t]emporary tables
       and tbl.relkind in ('r', 'v', 'f', 'p', 'm') -- o[r]dinary table, [v]iew, [f]oreign table, [p]artitioned table, [m]aterialized view. Other values are [i]ndex, [S]equence, [c]omposite type, [t]OAST table

docs/backlog.md

@@ -1,5 +1,9 @@
 # dbt-cratedb2 backlog
 
+## Core
+- Make write synchronization only optional?
+- Use psycopg3?
+
 ## Bugs?
 - [o] 'Database Error\n  Validation Failed: 1: this action would add [4] total shards, but this cluster currently has [1000]/[1000] maximum shards open;
    ```

pyproject.toml

@@ -133,51 +133,6 @@ env_files = ["test.env"]
 # FIXME: Make all test cases succeed, or specifically ignore individual ones.
 addopts = """
     -rfEXs -p pytester --strict-markers --verbosity=3
-    --ignore-glob=tests/functional/adapter/persist_docs/*
-    --ignore-glob=tests/functional/adapter/test_caching.py
-    --ignore-glob=tests/functional/adapter/test_clone.py
-    --ignore-glob=tests/functional/adapter/test_concurrency.py
-    --ignore-glob=tests/functional/adapter/test_constraints.py
-    --ignore-glob=tests/functional/adapter/test_empty.py
-    --ignore-glob=tests/functional/adapter/test_ephemeral.py
-    --ignore-glob=tests/functional/adapter/test_data_types.py
-    --ignore-glob=tests/functional/adapter/test_grants.py
-    --ignore-glob=tests/functional/adapter/test_incremental.py
-    --ignore-glob=tests/functional/adapter/test_incremental_microbatch.py
-    --ignore-glob=tests/functional/adapter/test_persist_docs.py
-    --ignore-glob=tests/functional/adapter/test_relations.py
-    --ignore-glob=tests/functional/adapter/test_show.py
-    --ignore-glob=tests/functional/adapter/test_simple_copy.py
-    --ignore-glob=tests/functional/adapter/test_simple_seed/*
-    --ignore-glob=tests/functional/adapter/test_simple_snapshot.py
-    --ignore-glob=tests/functional/adapter/test_store_test_failures.py
-    --ignore-glob=tests/functional/adapter/test_unit_testing.py
-    --ignore-glob=tests/functional/adapter/test_utils.py
-    --ignore-glob=tests/functional/contracts/*
-    --ignore-glob=tests/functional/custom_singular_tests/*
-    --ignore-glob=tests/functional/exit_codes/*
-    --ignore-glob=tests/functional/exposures/*
-    --ignore-glob=tests/functional/graph_selection/*
-    --ignore-glob=tests/functional/incremental_schema_tests/*
-    --ignore-glob=tests/functional/materializations/*
-    --ignore-glob=tests/functional/postgres/*
-    --ignore-glob=tests/functional/retry/*
-    --ignore-glob=tests/functional/run_operations/*
-    --ignore-glob=tests/functional/schema/*
-    --ignore-glob=tests/functional/selected_resources/*
-    --ignore-glob=tests/functional/semantic_models/*
-    --ignore-glob=tests/functional/show/*
-    --ignore-glob=tests/functional/sources/test_simple_source.py
-    --ignore-glob=tests/functional/statements/*
-    --ignore-glob=tests/functional/test_catalog.py
-    --ignore-glob=tests/functional/test_column_quotes.py
-    --ignore-glob=tests/functional/test_default_selectors.py
-    --ignore-glob=tests/functional/test_external_reference.py
-    --ignore-glob=tests/functional/test_multiple_indexes.py
-    --ignore-glob=tests/functional/test_ref_override.py
-    --ignore-glob=tests/functional/test_relation_name.py
-    --ignore-glob=tests/functional/test_severity.py
-    --ignore-glob=tests/functional/unit_testing/*
 """
 log_level = "DEBUG"
 log_cli_level = "DEBUG"

tests/functional/adapter/test_basic.py

@@ -28,7 +28,7 @@
 {% if execute %}
     {# don't ever do any of this #}
 
-    {# FIXME: CrateDB does not understand `DROP SCHEMA` yet #}
+    {# FIXME: CrateDB: `DROP SCHEMA` not supported #}
     {%- do adapter.drop_schema(upstream) -%}
 
     {# FIXME: CrateDB workaround #}

tests/functional/adapter/test_constraints.py

@@ -1,3 +1,4 @@
+import pytest
 from dbt.tests.adapter.constraints.test_constraints import (
     BaseTableConstraintsColumnsEqual,
     BaseViewConstraintsColumnsEqual,
@@ -11,49 +12,113 @@
     BaseModelConstraintsRuntimeEnforcement,
     BaseConstraintQuotedColumn,
     BaseIncrementalForeignKeyConstraint,
+    BaseConstraintsColumnsEqual,
 )
 
 
-class TestTableConstraintsColumnsEqual(BaseTableConstraintsColumnsEqual):
+class BaseConstraintsColumnsEqualCrateDB(BaseConstraintsColumnsEqual):
+
+    @pytest.fixture
+    def data_types(self, schema_int_type, int_type, string_type):
+        # sql_column_value, schema_data_type, error_data_type
+        return [
+            ["1", schema_int_type, int_type],
+            ["'1'", string_type, string_type],
+            # TODO: CrateDB: Cannot find data type: bool
+            # ["true", "bool", "BOOL"],
+            ["'2013-11-03 00:00:00-07'::timestamptz", "timestamptz", "DATETIMETZ"],
+            ["'2013-11-03 00:00:00-07'::timestamp", "timestamp", "DATETIME"],
+            ["ARRAY['a','b','c']", "text[]", "STRINGARRAY"],
+            ["ARRAY[1,2,3]", "int[]", "INTEGERARRAY"],
+            # TODO: CrateDB: NUMERIC storage is only supported if precision and scale are specified
+            # ["'1'::numeric", "numeric", "DECIMAL"],
+            # TODO: CrateDB: Type `json` does not support storage
+            # ["""'{"bar": "baz", "balance": 7.77, "active": false}'::json""", "json", "JSON"],
+        ]
+
+    @pytest.mark.skip("CrateDB: Does not work for unknown reasons")
+    def test__constraints_wrong_column_order(self, project):
+        pass
+
+    @pytest.mark.skip("CrateDB: Does not support foreign keys")
+    def test__constraints_wrong_column_names(self, project, string_type, int_type):
+        pass
+
+
+class TestTableConstraintsColumnsEqual(
+    BaseTableConstraintsColumnsEqual, BaseConstraintsColumnsEqualCrateDB
+):
     pass
 
 
-class TestViewConstraintsColumnsEqual(BaseViewConstraintsColumnsEqual):
+class TestViewConstraintsColumnsEqual(
+    BaseViewConstraintsColumnsEqual, BaseConstraintsColumnsEqualCrateDB
+):
     pass
 
 
-class TestIncrementalConstraintsColumnsEqual(BaseIncrementalConstraintsColumnsEqual):
-    pass
+class TestIncrementalConstraintsColumnsEqual(
+    BaseIncrementalConstraintsColumnsEqual, BaseConstraintsColumnsEqualCrateDB
+):
+    @pytest.mark.skip("CrateDB: Does not support foreign keys")
+    def test__constraints_ddl(self, project, expected_sql):
+        pass
 
 
-class TestTableConstraintsRuntimeDdlEnforcement(BaseConstraintsRuntimeDdlEnforcement):
-    pass
+class TestTableConstraintsRuntimeDdlEnforcement(
+    BaseConstraintsRuntimeDdlEnforcement, BaseConstraintsColumnsEqualCrateDB
+):
+    @pytest.mark.skip("CrateDB: Does not support foreign keys")
+    def test__constraints_ddl(self, project, expected_sql):
+        pass
 
 
-class TestTableConstraintsRollback(BaseConstraintsRollback):
-    pass
+class TestTableConstraintsRollback(BaseConstraintsRollback, BaseConstraintsColumnsEqualCrateDB):
+    @pytest.mark.skip("CrateDB: Does not work for unknown reasons")
+    def test__constraints_enforcement_rollback(
+        self, project, expected_color, expected_error_messages, null_model_sql
+    ):
+        pass
 
 
 class TestIncrementalConstraintsRuntimeDdlEnforcement(
-    BaseIncrementalConstraintsRuntimeDdlEnforcement
+    BaseIncrementalConstraintsRuntimeDdlEnforcement, BaseConstraintsColumnsEqualCrateDB
 ):
-    pass
+    @pytest.mark.skip("CrateDB: Does not support foreign keys")
+    def test__constraints_ddl(self, project, expected_sql):
+        pass
 
 
-class TestIncrementalConstraintsRollback(BaseIncrementalConstraintsRollback):
-    pass
+class TestIncrementalConstraintsRollback(
+    BaseIncrementalConstraintsRollback, BaseConstraintsColumnsEqualCrateDB
+):
+    @pytest.mark.skip("CrateDB: Does not work for unknown reasons")
+    def test__constraints_enforcement_rollback(
+        self, project, expected_color, expected_error_messages, null_model_sql
+    ):
+        pass
 
 
-class TestTableContractSqlHeader(BaseTableContractSqlHeader):
-    pass
+class TestTableContractSqlHeader(BaseTableContractSqlHeader, BaseConstraintsColumnsEqualCrateDB):
+    @pytest.mark.skip("CrateDB: Does not work for unknown reasons")
+    def test__contract_sql_header(self, project):
+        pass
 
 
-class TestIncrementalContractSqlHeader(BaseIncrementalContractSqlHeader):
-    pass
+class TestIncrementalContractSqlHeader(
+    BaseIncrementalContractSqlHeader, BaseConstraintsColumnsEqualCrateDB
+):
+    @pytest.mark.skip("CrateDB: Does not work for unknown reasons")
+    def test__contract_sql_header(self, project):
+        pass
 
 
-class TestModelConstraintsRuntimeEnforcement(BaseModelConstraintsRuntimeEnforcement):
-    pass
+class TestModelConstraintsRuntimeEnforcement(
+    BaseModelConstraintsRuntimeEnforcement, BaseConstraintsColumnsEqualCrateDB
+):
+    @pytest.mark.skip("CrateDB: Does not support foreign keys")
+    def test__model_constraints_ddl(self, project, expected_sql):
+        pass
 
 
 class TestConstraintQuotedColumn(BaseConstraintQuotedColumn):

tests/functional/adapter/test_data_types.py

@@ -1,3 +1,4 @@
+import pytest
 from dbt.tests.adapter.utils.data_types.test_type_bigint import BaseTypeBigInt
 from dbt.tests.adapter.utils.data_types.test_type_boolean import BaseTypeBoolean
 from dbt.tests.adapter.utils.data_types.test_type_float import BaseTypeFloat
@@ -15,6 +16,7 @@ class TestTypeBoolean(BaseTypeBoolean):
     pass
 
 
+@pytest.mark.skip("CrateDB: Type difference detected")
 class TestTypeFloat(BaseTypeFloat):
     pass
 

tests/functional/adapter/test_empty.py

@@ -1,5 +1,9 @@
+import pytest
 from dbt.tests.adapter.empty.test_empty import BaseTestEmpty
 
 
+@pytest.mark.skip(
+    "CrateDB: Couldn't create execution plan, see https://github.com/crate/crate/issues/17051"
+)
 class TestEmpty(BaseTestEmpty):
     pass