diff --git a/bft/dialects/types.py b/bft/dialects/types.py index f36ff58c..10d448ad 100644 --- a/bft/dialects/types.py +++ b/bft/dialects/types.py @@ -118,7 +118,6 @@ def __supports_case_kernel( args: List[CaseLiteral], result: CaseLiteral | Literal["error", "undefined"], ): - arg_len_matched = False # figure out if case is a supported kernel. walk over each supported kernel and check if any kernel # matches the case arguments type for supported_kernel in dfunc.supported_kernels: @@ -130,7 +129,6 @@ def __supports_case_kernel( arg_len = len(args) if len(supported_kernel.arg_types) != arg_len and dfunc.variadic_min == -1: continue - arg_len_matched = True matched = True kernel_arg_types = supported_kernel.arg_types if dfunc.variadic_min != -1 and len(supported_kernel.arg_types) == 1: @@ -161,8 +159,6 @@ def __supports_case_kernel( if matched: return None - if not arg_len_matched: - raise Exception("Unreachable path. Supported kernel with different # of types than case") return f"The dialect {self.name} does not support the kernel {case_to_kernel_str(dfunc.name, args, result)}" def __supports_options(self, dfunc: DialectFunction, case: Case): diff --git a/bft/tests/base.py b/bft/tests/base.py index 670f90d7..727c4a58 100644 --- a/bft/tests/base.py +++ b/bft/tests/base.py @@ -5,24 +5,42 @@ from bft.cases.parser import CaseFileParser from bft.cases.types import Case -from bft.dialects.types import DialectsLibrary from bft.testers.base_tester import BaseTester +from tools.convert_testcases.convert_testcases_to_yaml_format import ( + convert_directory as convert_directory_from_substrait, +) # Would be nice to have this as a session-scoped fixture but it doesn't seem that # parameter values can be a fixture def cases() -> List[Case]: cases = [] + bft_dir = Path(__file__).parent.parent.parent parser = CaseFileParser() - cases_dir = Path(__file__) / ".." / ".." / ".." / "cases" + cases_dir = bft_dir / "cases" + substrait_cases_dir = bft_dir / "substrait" / "tests" / "cases" + convert_directory_from_substrait(substrait_cases_dir, cases_dir) for case_path in cases_dir.resolve().rglob("*.yaml"): with open(case_path, "rb") as case_f: for case_file in parser.parse(case_f): for case in case_file.cases: + case = transform_case(case) cases.append(case) return cases +def transform_case(case): + # Create a new Case instance with updated `args` + return Case( + function=case.function, + base_uri=case.base_uri, + group=case.group, + args=case.args, # Update args here + result=case.result, + options=case.options, + ) + + def case_id_fn(case: Case): return f"{case.function}_{case.group.id}_{case.group.index}" diff --git a/ci/docker/base-tester.Dockerfile b/ci/docker/base-tester.Dockerfile index b33f13d3..f690f209 100644 --- a/ci/docker/base-tester.Dockerfile +++ b/ci/docker/base-tester.Dockerfile @@ -2,10 +2,11 @@ FROM alpine:3.18 ARG PIP_PACKAGES ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python RUN python3 -m ensurepip RUN echo "PIP_PACKAGES is $PIP_PACKAGES" -RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe $PIP_PACKAGES +RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe $PIP_PACKAGES ruamel.yaml antlr4-python3-runtime pytz WORKDIR /bft COPY . . diff --git a/ci/docker/datafusion.Dockerfile b/ci/docker/datafusion.Dockerfile index dadc927d..f109da67 100644 --- a/ci/docker/datafusion.Dockerfile +++ b/ci/docker/datafusion.Dockerfile @@ -1,9 +1,10 @@ FROM ubuntu:22.04 ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apt-get update && apt-get install -y python3.10 && ln -sf python3 /usr/bin/python RUN apt install -y pip -RUN pip install --upgrade pip setuptools pytest pyyaml mistletoe datafusion +RUN pip install --upgrade pip setuptools pytest pyyaml mistletoe datafusion ruamel.yaml antlr4-python3-runtime pytz numpy WORKDIR /bft COPY . . diff --git a/ci/docker/duckdb.Dockerfile b/ci/docker/duckdb.Dockerfile index 655392ca..9d8f5e44 100644 --- a/ci/docker/duckdb.Dockerfile +++ b/ci/docker/duckdb.Dockerfile @@ -1,9 +1,10 @@ FROM alpine:3.18 ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python RUN python3 -m ensurepip -RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe duckdb +RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe duckdb ruamel.yaml antlr4-python3-runtime pytz WORKDIR /bft COPY . . diff --git a/ci/docker/postgres-server.Dockerfile b/ci/docker/postgres-server.Dockerfile index 31a72fb7..2f46aba3 100644 --- a/ci/docker/postgres-server.Dockerfile +++ b/ci/docker/postgres-server.Dockerfile @@ -3,9 +3,10 @@ ENV POSTGRES_DB=bft ENV POSTGRES_PASSWORD=postgres ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python RUN python3 -m ensurepip -RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe psycopg[binary] +RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe psycopg[binary] ruamel.yaml antlr4-python3-runtime pytz WORKDIR /bft COPY . . diff --git a/ci/docker/sqlite.Dockerfile b/ci/docker/sqlite.Dockerfile index 4ee38811..b41e838d 100644 --- a/ci/docker/sqlite.Dockerfile +++ b/ci/docker/sqlite.Dockerfile @@ -1,11 +1,13 @@ FROM alpine:3.18 ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apk add --update --no-cache python3 && ln -sf python3 /usr/bin/python RUN python3 -m ensurepip -RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe +RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe ruamel.yaml antlr4-python3-runtime pytz WORKDIR /bft COPY . . +# CMD to run all commands and display the results CMD /usr/bin/python -mpytest bft/tests/test_sqlite.py diff --git a/ci/docker/velox.Dockerfile b/ci/docker/velox.Dockerfile index 448082c2..f3116f08 100644 --- a/ci/docker/velox.Dockerfile +++ b/ci/docker/velox.Dockerfile @@ -1,10 +1,11 @@ FROM ubuntu:22.04 ENV PYTHONUNBUFFERED=1 +ENV PYTHONPATH=/bft/substrait RUN apt-get update && apt-get install -y \ python3 \ python3-pip -RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe pyvelox +RUN pip3 install --no-cache --upgrade pip setuptools pytest pyyaml mistletoe pyvelox ruamel.yaml antlr4-python3-runtime pytz WORKDIR /bft COPY . . diff --git a/dialects/datafusion.yaml b/dialects/datafusion.yaml index 9ee461d3..2ea53652 100644 --- a/dialects/datafusion.yaml +++ b/dialects/datafusion.yaml @@ -441,8 +441,6 @@ scalar_functions: required_options: lookaround: false supported_kernels: - - str_str_str_i64_i64 - - vchar_vchar_vchar_i64_i64 - str_str_str - name: string.bit_length supported_kernels: @@ -587,8 +585,3 @@ aggregate_functions: aggregate: true supported_kernels: - '' -- name: aggregate_approx.approx_count_distinct - local_name: approx_distinct - aggregate: true - supported_kernels: - - any diff --git a/dialects/duckdb.yaml b/dialects/duckdb.yaml index a8e4f809..cde670c1 100644 --- a/dialects/duckdb.yaml +++ b/dialects/duckdb.yaml @@ -498,8 +498,6 @@ scalar_functions: required_options: lookaround: false supported_kernels: - - str_str_str_i64_i64 - - vchar_vchar_vchar_i64_i64 - str_str_str - name: string.regexp_string_split local_name: regexp_split_to_array @@ -513,8 +511,6 @@ scalar_functions: required_options: lookaround: false supported_kernels: - - vchar_vchar_i64_i64_i64 - - str_str_i64_i64_i64 - str_str - name: string.bit_length supported_kernels: diff --git a/dialects/postgres.yaml b/dialects/postgres.yaml index 4fd80a3a..d2c254bb 100644 --- a/dialects/postgres.yaml +++ b/dialects/postgres.yaml @@ -530,8 +530,6 @@ scalar_functions: required_options: lookaround: true supported_kernels: - - str_str_str_i64_i64 - - vchar_vchar_vchar_i64_i64 - str_str_str - name: string.regexp_string_split local_name: regexp_split_to_array @@ -541,15 +539,10 @@ scalar_functions: - name: string.regexp_count_substring local_name: regexp_count supported_kernels: - - str_str_i64 - - vchar_vchar_i64 - - fchar_fchar_i64 - str_str - name: string.regexp_match_substring local_name: regexp_substr supported_kernels: - - vchar_vchar_i64_i64_i64 - - str_str_i64_i64_i64 - str_str - name: string.bit_length supported_kernels: diff --git a/dialects/velox_presto.yaml b/dialects/velox_presto.yaml index afddeb1d..27278760 100644 --- a/dialects/velox_presto.yaml +++ b/dialects/velox_presto.yaml @@ -325,27 +325,6 @@ scalar_functions: - str - vchar - fchar -- name: string.ltrim - required_options: - spaces_only: true - supported_kernels: - - vchar_vchar - - str_str - - str -- name: string.rtrim - required_options: - spaces_only: true - supported_kernels: - - vchar_vchar - - str_str - - str -- name: string.trim - required_options: - spaces_only: true - supported_kernels: - - vchar_vchar - - str_str - - str - name: string.lpad supported_kernels: - vchar_i32_vchar