Run tests from substrait

1. Fix dialects to only contain the supported functions
2. Don't raise exception if a test case's arguments don't match with
the supported kernel. It is not a failure case. Examples can include
regexp string functions.
3. Update Dockerfiles to install the required packages

bft/dialects/types.py

@@ -118,7 +118,6 @@ def __supports_case_kernel(
             args: List[CaseLiteral],
             result: CaseLiteral | Literal["error", "undefined"],
     ):
-        arg_len_matched = False
         # figure out if case is a supported kernel. walk over each supported kernel and check if any kernel
         # matches the case arguments type
         for supported_kernel in dfunc.supported_kernels:
@@ -130,7 +129,6 @@ def __supports_case_kernel(
                 arg_len = len(args)
             if len(supported_kernel.arg_types) != arg_len and dfunc.variadic_min == -1:
                 continue
-            arg_len_matched = True
             matched = True
             kernel_arg_types = supported_kernel.arg_types
             if dfunc.variadic_min != -1 and len(supported_kernel.arg_types) == 1:
@@ -161,8 +159,6 @@ def __supports_case_kernel(
             if matched:
                 return None
 
-        if not arg_len_matched:
-            raise Exception("Unreachable path.  Supported kernel with different # of types than case")
         return f"The dialect {self.name} does not support the kernel {case_to_kernel_str(dfunc.name, args, result)}"
 
     def __supports_options(self, dfunc: DialectFunction, case: Case):

bft/tests/base.py

@@ -5,24 +5,42 @@
 
 from bft.cases.parser import CaseFileParser
 from bft.cases.types import Case
-from bft.dialects.types import DialectsLibrary
 from bft.testers.base_tester import BaseTester
+from tools.convert_testcases.convert_testcases_to_yaml_format import (
+    convert_directory as convert_directory_from_substrait,
+)
 
 
 # Would be nice to have this as a session-scoped fixture but it doesn't seem that
 # parameter values can be a fixture
 def cases() -> List[Case]:
     cases = []
+    bft_dir = Path(__file__).parent.parent.parent
     parser = CaseFileParser()
-    cases_dir = Path(__file__) / ".." / ".." / ".." / "cases"
+    cases_dir = bft_dir / "cases"
+    substrait_cases_dir = bft_dir / "substrait" / "tests" / "cases"
+    convert_directory_from_substrait(substrait_cases_dir, cases_dir)
     for case_path in cases_dir.resolve().rglob("*.yaml"):
         with open(case_path, "rb") as case_f:
             for case_file in parser.parse(case_f):
                 for case in case_file.cases:
+                    case = transform_case(case)
                     cases.append(case)
     return cases
 
 
+def transform_case(case):
+    # Create a new Case instance with updated `args`
+    return Case(
+        function=case.function,
+        base_uri=case.base_uri,
+        group=case.group,
+        args=case.args,  # Update args here
+        result=case.result,
+        options=case.options,
+    )
+
+
 def case_id_fn(case: Case):
     return f"{case.function}_{case.group.id}_{case.group.index}"
 

ci/docker/base-tester.Dockerfile

@@ -2,10 +2,11 @@ FROM alpine:3.18
 ARG PIP_PACKAGES
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
 RUN python3 -m ensurepip
 RUN echo "PIP_PACKAGES is $PIP_PACKAGES"
-RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe $PIP_PACKAGES
+RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe $PIP_PACKAGES ruamel.yaml antlr4-python3-runtime pytz
 
 WORKDIR /bft
 COPY . .

ci/docker/datafusion.Dockerfile

@@ -1,9 +1,10 @@
 FROM ubuntu:22.04
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apt-get update && apt-get install -y python3.10 && ln -sf python3 /usr/bin/python
 RUN apt install -y pip
-RUN pip install --upgrade pip setuptools pytest pyyaml mistletoe datafusion
+RUN pip install --upgrade pip setuptools pytest pyyaml mistletoe datafusion ruamel.yaml antlr4-python3-runtime pytz numpy
 
 WORKDIR /bft
 COPY . .

ci/docker/duckdb.Dockerfile

@@ -1,9 +1,10 @@
 FROM alpine:3.18
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
 RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe duckdb
+RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe duckdb ruamel.yaml antlr4-python3-runtime pytz
 
 WORKDIR /bft
 COPY . .

ci/docker/postgres-server.Dockerfile

@@ -3,9 +3,10 @@ ENV POSTGRES_DB=bft
 ENV POSTGRES_PASSWORD=postgres
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
 RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe psycopg[binary]
+RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe psycopg[binary] ruamel.yaml antlr4-python3-runtime pytz
 
 WORKDIR /bft
 COPY . .

ci/docker/sqlite.Dockerfile

@@ -1,11 +1,13 @@
 FROM alpine:3.18
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python
 RUN python3 -m ensurepip
-RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe
+RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe ruamel.yaml antlr4-python3-runtime pytz
 
 WORKDIR /bft
 COPY . .
 
+# CMD to run all commands and display the results
 CMD /usr/bin/python -mpytest bft/tests/test_sqlite.py

ci/docker/velox.Dockerfile

@@ -1,10 +1,11 @@
 FROM ubuntu:22.04
 
 ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/bft/substrait
 RUN apt-get update && apt-get install -y \
     python3 \
     python3-pip
-RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe pyvelox
+RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe pyvelox ruamel.yaml antlr4-python3-runtime pytz
 
 WORKDIR /bft
 COPY . .

dialects/datafusion.yaml

@@ -441,8 +441,6 @@ scalar_functions:
   required_options:
     lookaround: false
   supported_kernels:
-  - str_str_str_i64_i64
-  - vchar_vchar_vchar_i64_i64
   - str_str_str
 - name: string.bit_length
   supported_kernels:
@@ -587,8 +585,3 @@ aggregate_functions:
   aggregate: true
   supported_kernels:
   - ''
-- name: aggregate_approx.approx_count_distinct
-  local_name: approx_distinct
-  aggregate: true
-  supported_kernels:
-  - any

dialects/duckdb.yaml

@@ -498,8 +498,6 @@ scalar_functions:
   required_options:
     lookaround: false
   supported_kernels:
-  - str_str_str_i64_i64
-  - vchar_vchar_vchar_i64_i64
   - str_str_str
 - name: string.regexp_string_split
   local_name: regexp_split_to_array
@@ -513,8 +511,6 @@ scalar_functions:
   required_options:
     lookaround: false
   supported_kernels:
-  - vchar_vchar_i64_i64_i64
-  - str_str_i64_i64_i64
   - str_str
 - name: string.bit_length
   supported_kernels:

dialects/postgres.yaml

@@ -530,8 +530,6 @@ scalar_functions:
   required_options:
     lookaround: true
   supported_kernels:
-  - str_str_str_i64_i64
-  - vchar_vchar_vchar_i64_i64
   - str_str_str
 - name: string.regexp_string_split
   local_name: regexp_split_to_array
@@ -541,15 +539,10 @@ scalar_functions:
 - name: string.regexp_count_substring
   local_name: regexp_count
   supported_kernels:
-  - str_str_i64
-  - vchar_vchar_i64
-  - fchar_fchar_i64
   - str_str
 - name: string.regexp_match_substring
   local_name: regexp_substr
   supported_kernels:
-  - vchar_vchar_i64_i64_i64
-  - str_str_i64_i64_i64
   - str_str
 - name: string.bit_length
   supported_kernels:

dialects/velox_presto.yaml

@@ -325,27 +325,6 @@ scalar_functions:
   - str
   - vchar
   - fchar
-- name: string.ltrim
-  required_options:
-    spaces_only: true
-  supported_kernels:
-  - vchar_vchar
-  - str_str
-  - str
-- name: string.rtrim
-  required_options:
-    spaces_only: true
-  supported_kernels:
-  - vchar_vchar
-  - str_str
-  - str
-- name: string.trim
-  required_options:
-    spaces_only: true
-  supported_kernels:
-  - vchar_vchar
-  - str_str
-  - str
 - name: string.lpad
   supported_kernels:
   - vchar_i32_vchar