Implement native conformance test runner

WORKSPACE

@@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 http_archive(
     name = "io_bazel_rules_go",
-    sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
+    sha256 = "56d8c5a5c91e1af73eca71a6fab2ced959b67c86d12ba37feedb0a2dfea441a6",
     urls = [
-        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
-        "https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip",
+        "https://github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip",
     ],
 )
 
@@ -155,6 +155,13 @@ go_repository(
     version = "v0.0.0-20220722155223-a9213eeb770e",
 )
 
+go_repository(
+    name = "com_github_google_go_cmp",
+    importpath = "github.com/google/go-cmp",
+    sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=",
+    version = "v0.5.9",
+)
+
 # Run the dependencies at the end.  These will silently try to import some
 # of the above repositories but at different versions, so ours must come first.
 go_rules_dependencies()

conformance/BUILD.bazel

@@ -1,4 +1,11 @@
-ALL_TESTS = [
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+load("//conformance:conformance_test.bzl", "conformance_test")
+
+package(
+    licenses = ["notice"],  # Apache 2.0
+)
+
+_ALL_TESTS = [
     "@com_google_cel_spec//tests/simple:testdata/basic.textproto",
     "@com_google_cel_spec//tests/simple:testdata/comparisons.textproto",
     "@com_google_cel_spec//tests/simple:testdata/conversions.textproto",
@@ -24,47 +31,60 @@ ALL_TESTS = [
     "@com_google_cel_spec//tests/simple:testdata/wrappers.textproto",
 ]
 
-sh_test(
-    name = "ct",
-    srcs = ["@com_google_cel_spec//tests:conftest.sh"],
-    args = [
-        "$(location @com_google_cel_spec//tests/simple:simple_test)",
-        "--server=$(location //server/main:cel_server)",
-        # Tests that need to be removed as the spec has changed
-        "--skip_test=comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error;ne_literal/ne_mixed_types_error",
-        "--skip_test=comparisons/in_list_literal/elem_in_mixed_type_list_error",
-        "--skip_test=comparisons/in_map_literal/key_in_mixed_key_type_map_error",
-        "--skip_test=macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive",
+_TESTS_TO_SKIP = [
+    "comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error",
+    "comparisons/ne_literal/ne_mixed_types_error",
+    "comparisons/in_list_literal/elem_in_mixed_type_list_error",
+    "comparisons/in_map_literal/key_in_mixed_key_type_map_error",
+    "macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive",
 
-        # Failing conformance tests.
-        "--skip_test=fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key",
-        "--skip_test=fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous",
-        "--skip_test=macros/map/map_extract_keys",
-        "--skip_test=timestamps/duration_converters/get_milliseconds",
+    # Failing conformance tests.
+    "fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key",
+    "fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous",
+    "macros/map/map_extract_keys",
+    "timestamps/duration_converters/get_milliseconds",
 
-        # Future enhancments.
-        "--skip_test=enums/strong_proto2",
-        "--skip_test=enums/strong_proto3",
-    ] + ["$(location " + test + ")" for test in ALL_TESTS],
-    data = [
-        "//server/main:cel_server",
-        "@com_google_cel_spec//tests/simple:simple_test",
-    ] + ALL_TESTS,
+    # Future enhancments.
+    "enums/strong_proto2",
+    "enums/strong_proto3",
+]
+
+go_test(
+    name = "go_default_test",
+    size = "small",
+    srcs = [
+        "conformance_test.go",
+    ],
+    tags = [
+        "manual",
+        "notap",
+    ],
+    deps = [
+        "//cel:go_default_library",
+        "//common:go_default_library",
+        "//common/types:go_default_library",
+        "//common/types/ref:go_default_library",
+        "//ext:go_default_library",
+        "@com_github_google_go_cmp//cmp:go_default_library",
+        "@com_google_cel_spec//proto/test/v1:testpb_go_proto",
+        "@com_google_cel_spec//proto/test/v1/proto2:test_all_types_go_proto",
+        "@com_google_cel_spec//proto/test/v1/proto3:test_all_types_go_proto",
+        "@io_bazel_rules_go//go/runfiles",
+        "@org_golang_google_genproto_googleapis_api//expr/v1alpha1:go_default_library",
+        "@org_golang_google_protobuf//encoding/prototext:go_default_library",
+        "@org_golang_google_protobuf//testing/protocmp:go_default_library",
+    ],
 )
 
-# ct_dashboard is a target for the conformance dashboard and includes all simple textproto files, including those that are broken.
-sh_test(
-    name = "ct_dashboard",
-    srcs = ["@com_google_cel_spec//tests:conftest-nofail.sh"],
-    args = [
-        "$(location @com_google_cel_spec//tests/simple:simple_test)",
-        "--server=$(location //server/main:cel_server)",
+conformance_test(
+    name = "conformance",
+    dashboard = False,
+    data = _ALL_TESTS,
+    skip_tests = _TESTS_TO_SKIP,
+)
 
-        # Failing due to a GCB builder issue
-        "--skip_test=timestamps/timestamp_selectors_tz/getDate,getDayOfMonth_name_neg,getDayOfMonth_name_pos,getDayOfYear,getMinutes",
-    ] + ["$(location " + test + ")" for test in ALL_TESTS],
-    data = [
-        "//server/main:cel_server",
-        "@com_google_cel_spec//tests/simple:simple_test",
-    ] + ALL_TESTS,
+conformance_test(
+    name = "conformance_dashboard",
+    dashboard = True,
+    data = _ALL_TESTS,
 )

conformance/conformance_test.bzl

@@ -0,0 +1,58 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+This module contains build rules for generating the conformance test targets.
+"""
+
+# Converts the list of tests to skip from the format used by the original Go test runner to a single
+# flag value where each test is separated by a comma. It also performs expansion, for example
+# `foo/bar,baz` becomes two entries which are `foo/bar` and `foo/baz`.
+def _expand_tests_to_skip(tests_to_skip):
+    result = []
+    for test_to_skip in tests_to_skip:
+        comma = test_to_skip.find(",")
+        if comma == -1:
+            result.append(test_to_skip)
+            continue
+        slash = test_to_skip.rfind("/", 0, comma)
+        if slash == -1:
+            slash = 0
+        else:
+            slash = slash + 1
+        for part in test_to_skip[slash:].split(","):
+            result.append(test_to_skip[0:slash] + part)
+    return result
+
+def _conformance_test_args(data, skip_tests, dashboard):
+    args = []
+    args.append("--skip_tests={}".format(",".join(_expand_tests_to_skip(skip_tests))))
+    args.append("--tests={}".format(",".join(["$(location " + test + ")" for test in data])))
+    if dashboard:
+        args.append("--dashboard")
+    return args
+
+def conformance_test(name, data, dashboard, skip_tests = []):
+    native.sh_test(
+        name = name,
+        size = "small",
+        srcs = ["//conformance:conformance_test.sh"],
+        args = ["$(location //conformance:go_default_test)"] + _conformance_test_args(data, skip_tests, dashboard),
+        data = ["//conformance:go_default_test"] + data,
+        tags = [
+            "guitar",
+            "manual",
+            "notap",
+        ] if dashboard else [],
+    )