Skip to content

Commit

Permalink
Implement native conformance test runner
Browse files Browse the repository at this point in the history
Signed-off-by: Justin King <jcking@google.com>
  • Loading branch information
jcking committed Aug 12, 2024
1 parent 851a4b8 commit a799e37
Show file tree
Hide file tree
Showing 9 changed files with 447 additions and 125 deletions.
13 changes: 10 additions & 3 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")

http_archive(
name = "io_bazel_rules_go",
sha256 = "099a9fb96a376ccbbb7d291ed4ecbdfd42f6bc822ab77ae6f1b5cb9e914e94fa",
sha256 = "56d8c5a5c91e1af73eca71a6fab2ced959b67c86d12ba37feedb0a2dfea441a6",
urls = [
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.35.0/rules_go-v0.35.0.zip",
"https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip",
"https://github.com/bazelbuild/rules_go/releases/download/v0.37.0/rules_go-v0.37.0.zip",
],
)

Expand Down Expand Up @@ -155,6 +155,13 @@ go_repository(
version = "v0.0.0-20220722155223-a9213eeb770e",
)

go_repository(
name = "com_github_google_go_cmp",
importpath = "github.com/google/go-cmp",
sum = "h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=",
version = "v0.5.9",
)

# Run the dependencies at the end. These will silently try to import some
# of the above repositories but at different versions, so ours must come first.
go_rules_dependencies()
Expand Down
98 changes: 59 additions & 39 deletions conformance/BUILD.bazel
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
ALL_TESTS = [
load("@io_bazel_rules_go//go:def.bzl", "go_test")
load("//conformance:conformance_test.bzl", "conformance_test")

package(
licenses = ["notice"], # Apache 2.0
)

_ALL_TESTS = [
"@com_google_cel_spec//tests/simple:testdata/basic.textproto",
"@com_google_cel_spec//tests/simple:testdata/comparisons.textproto",
"@com_google_cel_spec//tests/simple:testdata/conversions.textproto",
Expand All @@ -24,47 +31,60 @@ ALL_TESTS = [
"@com_google_cel_spec//tests/simple:testdata/wrappers.textproto",
]

sh_test(
name = "ct",
srcs = ["@com_google_cel_spec//tests:conftest.sh"],
args = [
"$(location @com_google_cel_spec//tests/simple:simple_test)",
"--server=$(location //server/main:cel_server)",
# Tests that need to be removed as the spec has changed
"--skip_test=comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error;ne_literal/ne_mixed_types_error",
"--skip_test=comparisons/in_list_literal/elem_in_mixed_type_list_error",
"--skip_test=comparisons/in_map_literal/key_in_mixed_key_type_map_error",
"--skip_test=macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive",
_TESTS_TO_SKIP = [
"comparisons/eq_literal/eq_mixed_types_error,eq_list_elem_mixed_types_error,eq_map_value_mixed_types_error;ne_literal/ne_mixed_types_error",
"comparisons/in_list_literal/elem_in_mixed_type_list_error",
"comparisons/in_map_literal/key_in_mixed_key_type_map_error",
"macros/exists/list_elem_type_exhaustive,map_key_type_exhaustive",

# Failing conformance tests.
"--skip_test=fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key",
"--skip_test=fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous",
"--skip_test=macros/map/map_extract_keys",
"--skip_test=timestamps/duration_converters/get_milliseconds",
# Failing conformance tests.
"fields/qualified_identifier_resolution/map_key_float,map_key_null,map_value_repeat_key",
"fields/qualified_identifier_resolution/map_value_repeat_key_heterogeneous",
"macros/map/map_extract_keys",
"timestamps/duration_converters/get_milliseconds",

# Future enhancments.
"--skip_test=enums/strong_proto2",
"--skip_test=enums/strong_proto3",
] + ["$(location " + test + ")" for test in ALL_TESTS],
data = [
"//server/main:cel_server",
"@com_google_cel_spec//tests/simple:simple_test",
] + ALL_TESTS,
# Future enhancments.
"enums/strong_proto2",
"enums/strong_proto3",
]

go_test(
name = "go_default_test",
size = "small",
testonly = True,
srcs = [
"conformance_test.go",
],
tags = [
"manual",
"notap",
],
deps = [
"//cel:go_default_library",
"//common:go_default_library",
"//common/types:go_default_library",
"//common/types/ref:go_default_library",
"//ext:go_default_library",
"@com_github_google_go_cmp//cmp:go_default_library",
"@com_google_cel_spec//proto/test/v1:testpb_go_proto",
"@com_google_cel_spec//proto/test/v1/proto2:test_all_types_go_proto",
"@com_google_cel_spec//proto/test/v1/proto3:test_all_types_go_proto",
"@io_bazel_rules_go//go/runfiles",
"@org_golang_google_genproto_googleapis_api//expr/v1alpha1:go_default_library",
"@org_golang_google_protobuf//encoding/prototext:go_default_library",
"@org_golang_google_protobuf//testing/protocmp:go_default_library",
],
)

# ct_dashboard is a target for the conformance dashboard and includes all simple textproto files, including those that are broken.
sh_test(
name = "ct_dashboard",
srcs = ["@com_google_cel_spec//tests:conftest-nofail.sh"],
args = [
"$(location @com_google_cel_spec//tests/simple:simple_test)",
"--server=$(location //server/main:cel_server)",
conformance_test(
name = "conformance",
dashboard = False,
data = _ALL_TESTS,
skip_tests = _TESTS_TO_SKIP,
)

# Failing due to a GCB builder issue
"--skip_test=timestamps/timestamp_selectors_tz/getDate,getDayOfMonth_name_neg,getDayOfMonth_name_pos,getDayOfYear,getMinutes",
] + ["$(location " + test + ")" for test in ALL_TESTS],
data = [
"//server/main:cel_server",
"@com_google_cel_spec//tests/simple:simple_test",
] + ALL_TESTS,
conformance_test(
name = "conformance_dashboard",
dashboard = True,
data = _ALL_TESTS,
)
58 changes: 58 additions & 0 deletions conformance/conformance_test.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
This module contains build rules for generating the conformance test targets.
"""

# Converts the list of tests to skip from the format used by the original Go test runner to a single
# flag value where each test is separated by a comma. It also performs expansion, for example
# `foo/bar,baz` becomes two entries which are `foo/bar` and `foo/baz`.
def _expand_tests_to_skip(tests_to_skip):
result = []
for test_to_skip in tests_to_skip:
comma = test_to_skip.find(",")
if comma == -1:
result.append(test_to_skip)
continue
slash = test_to_skip.rfind("/", 0, comma)
if slash == -1:
slash = 0
else:
slash = slash + 1
for part in test_to_skip[slash:].split(","):
result.append(test_to_skip[0:slash] + part)
return result

def _conformance_test_args(data, skip_tests, dashboard):
args = []
args.append("--skip_tests={}".format(",".join(_expand_tests_to_skip(skip_tests))))
args.append("--tests={}".format(",".join(["$(location " + test + ")" for test in data])))
if dashboard:
args.append("--dashboard")
return args

def conformance_test(name, data, dashboard, skip_tests = []):
native.sh_test(
name = name,
size = "small",
srcs = ["//conformance:conformance_test.sh"],
args = ["$(location //conformance:go_default_test)"] + _conformance_test_args(data, skip_tests, dashboard),
data = ["//conformance:go_default_test"] + data,
tags = [
"guitar",
"manual",
"notap",
] if dashboard else [],
)
Loading

0 comments on commit a799e37

Please sign in to comment.