Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bazel: add bazel mirror #39072

Merged
merged 8 commits into from
Jul 18, 2023
Merged

bazel: add bazel mirror #39072

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
1b3fc13
tools: add mysql-tester for adding bazel into mysql-test
hawkingrei Nov 10, 2022
52c73e9
update
hawkingrei Jul 17, 2023
d0d8872
update
hawkingrei Jul 18, 2023
173e11b
update
hawkingrei Jul 18, 2023
4bd3c22
update
hawkingrei Jul 18, 2023
e8741a6
remove useless
hawkingrei Jul 18, 2023
e70891e
update
hawkingrei Jul 18, 2023
beda35f
update
hawkingrei Jul 18, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,12 @@
load("@bazel_gazelle//:def.bzl", "gazelle")

exports_files([
"DEPS.bzl",
"WORKSPACE",
"go.mod",
"go.sum",
])

# gazelle:proto disable_global
# gazelle:prefix github.com/pingcap/tidb
# gazelle:exclude tidb-binlog/proto/proto
Expand Down
8,954 changes: 6,785 additions & 2,169 deletions DEPS.bzl
View file
Open in desktop

Large diffs are not rendered by default.

17 changes: 16 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,8 @@ generate_grafana_scripts:
bazel_ci_prepare:
bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //:gazelle
bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //:gazelle -- update-repos -from_file=go.mod -to_macro DEPS.bzl%go_deps -build_file_proto_mode=disable
bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //cmd/mirror:mirror -- --mirror> tmp.txt
mv tmp.txt DEPS.bzl
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe something like $tmp=$(mktemp bzltmp.XXXXX) would be better and not leave temp files around? Otherwise we might to add tmp.txt to the .gitignore.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks like no all > tmp.txt lines were replaced?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think what you mentioned is "bazel_ci_prepare". It was actually missed, but it's not a big problem because it only runs in CI.

bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) \
--run_under="cd $(CURDIR) && " \
//tools/tazel:tazel
Expand All @@ -448,6 +450,10 @@ bazel_prepare:
bazel run \
--run_under="cd $(CURDIR) && " \
//tools/tazel:tazel
$(eval $@TMP_OUT := $(shell mktemp -d -t tidbbzl.XXXXXX))
bazel run //cmd/mirror -- --mirror> $($@TMP_OUT)/tmp.txt
cp $($@TMP_OUT)/tmp.txt DEPS.bzl
rm -rf $($@TMP_OUT)

bazel_ci_prepare_rbe:
bazel run //:gazelle
Expand All @@ -473,7 +479,7 @@ bazel_coverage_test: check-bazel-prepare failpoint-enable bazel_ci_prepare
-- //... -//cmd/... -//tests/graceshutdown/... \
-//tests/globalkilltest/... -//tests/readonlytest/... -//br/pkg/task:task_test -//tests/realtikvtest/...

bazel_build: bazel_ci_prepare
bazel_build:
mkdir -p bin
bazel $(BAZEL_GLOBAL_CONFIG) build $(BAZEL_CMD_CONFIG) \
//... --//build:with_nogo_flag=true
Expand Down Expand Up @@ -555,3 +561,12 @@ docker:

docker-test:
docker buildx build --platform linux/amd64,linux/arm64 --push -t "$(DOCKERPREFIX)tidb:latest" --build-arg 'GOPROXY=$(shell go env GOPROXY),' -f Dockerfile .

bazel_mirror:
$(eval $@TMP_OUT := $(shell mktemp -d -t tidbbzl.XXXXXX))
bazel $(BAZEL_GLOBAL_CONFIG) run $(BAZEL_CMD_CONFIG) //cmd/mirror:mirror -- --mirror> $($@TMP_OUT)/tmp.txt
cp $($@TMP_OUT)/tmp.txt DEPS.bzl
rm -rf $($@TMP_OUT)

bazel_sync:
bazel $(BAZEL_GLOBAL_CONFIG) sync $(BAZEL_SYNC_CONFIG)
1 change: 1 addition & 0 deletions Makefile.common
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,4 +122,5 @@ TEST_COVERAGE_DIR := "test_coverage"
ifneq ("$(CI)", "")
BAZEL_GLOBAL_CONFIG := --output_user_root=/home/jenkins/.tidb/tmp
BAZEL_CMD_CONFIG := --config=ci --repository_cache=/home/jenkins/.tidb/tmp
BAZEL_SYNC_CONFIG := --repository_cache=/home/jenkins/.tidb/tmp
endif
4 changes: 4 additions & 0 deletions build/nogo_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@
"exclude_files": {
"parser/parser.go": "parser/parser.go code",
"external/": "no need to vet third party code",
".cgo/": "no need to cgo code",
".*_generated\\.go$": "ignore generated code",
".*_/testmain\\.go$": "ignore code"
}
Expand Down Expand Up @@ -278,6 +279,7 @@
"/cgo/": "ignore cgo code",
"/rules_go_work-*": "ignore generated code",
"tools/check/ut.go": "ignore tools/check code",
"cmd/mirror": "ignore cmd/mirror code",
"tools/check/xprog.go": "ignore tools/check code",
"cmd/pluginpkg/pluginpkg.go": "ignore cmd/pluginpkg code",
"tools/check/xprog.go:": "ignore tools/check code",
Expand Down Expand Up @@ -1138,6 +1140,7 @@
"prealloc": {
"exclude_files": {
"parser/parser.go": "parser/parser.go code",
"cmd/mirror": "cmd/mirror code",
"external/": "no need to vet third party code",
".*_generated\\.go$": "ignore generated code",
"parser/yy_parser.go": "ignore generated code",
Expand All @@ -1147,6 +1150,7 @@
"predeclared": {
"exclude_files": {
"external/": "no need to vet third party code",
"cmd/mirror": "no need to mirror",
".*_generated\\.go$": "ignore generated code",
"parser/yy_parser.go": "ignore generated code",
"parser/parser.go": "ignore generated code",
Expand Down
2 changes: 1 addition & 1 deletion build/patches/io_etcd_go_etcd_api_v3.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ diff -uprN old/io_etcd_go_etcd_api_v3/etcdserverpb/BUILD.bazel new/io_etcd_go_et
"@com_github_gogo_protobuf//gogoproto",
"@com_github_golang_protobuf//proto:go_default_library",
- "@go_googleapis//google/api:annotations_go_proto",
+ "@org_golang_google_genproto_googleapis_api//annotations:annotations",
+ "@org_golang_google_genproto//googleapis/api/annotations:annotations",
"@org_golang_google_grpc//:go_default_library",
"@org_golang_google_grpc//codes",
"@org_golang_google_grpc//status",
Expand Down
34 changes: 34 additions & 0 deletions cmd/mirror/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")

go_library(
name = "mirror_lib",
srcs = [
"mirror.go",
"skylarkutil.go",
],
importpath = "github.com/pingcap/tidb/cmd/mirror",
visibility = ["//visibility:private"],
deps = [
"@com_github_google_skylark//syntax",
"@com_google_cloud_go_storage//:storage",
"@io_bazel_rules_go//go/tools/bazel:go_default_library",
"@org_golang_google_api//googleapi",
"@org_golang_x_exp//slices",
"@org_golang_x_sync//errgroup",
],
)

go_binary(
name = "mirror",
data = [
"//:DEPS.bzl",
"//:go.mod",
"//:go.sum",
"//build/patches:all_patches",
"//parser:go.mod",
"//parser:go.sum",
"@go_sdk//:files",
],
embed = [":mirror_lib"],
visibility = ["//visibility:public"],
)
Loading