From 5bed8784b98dbb974773f264722cf530aeccaa9b Mon Sep 17 00:00:00 2001 From: Martin Taillefer Date: Wed, 25 Sep 2019 06:56:54 -0700 Subject: [PATCH] Update common files. (#5052) --- .gitattributes | 9 +++++ Makefile | 58 ++++++++++++++++------------- common/.commonfiles.sha | 2 +- common/Makefile.common.mk | 27 +++++++++----- common/config/.golangci.yml | 8 +++- common/config/mdl.rb | 2 +- common/scripts/gobuild.sh | 17 +++++++-- common/scripts/lint_go.sh | 24 ++++++++++++ common/scripts/report_build_info.sh | 31 +++++++-------- mdl.rb | 12 ++++++ scripts/lint_site.sh | 2 +- 11 files changed, 131 insertions(+), 61 deletions(-) create mode 100644 .gitattributes create mode 100755 common/scripts/lint_go.sh create mode 100644 mdl.rb diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000000..4ccc0f5d8d59 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,9 @@ +*.descriptor linguist-generated=true +*.descriptor -diff -merge +*.descriptor_set linguist-generated=true +*.descriptor_set -diff -merge +*.pb.html linguist-generated=true +*.pb.go linguist-generated=true +*.gen.go linguist-generated=true +*.gen.yaml linguist-generated=true +*_pb2.py linguist-generated=true diff --git a/Makefile b/Makefile index 93a604add717..429ed73a2b06 100644 --- a/Makefile +++ b/Makefile @@ -28,62 +28,68 @@ # figure out all the tools you need in your environment to make that work. export BUILD_WITH_CONTAINER ?= 0 -ifeq ($(BUILD_WITH_CONTAINER),1) -IMG = gcr.io/istio-testing/build-tools:2019-08-29T13-57-48 -UID = $(shell id -u) -PWD = $(shell pwd) -GOBIN_SOURCE ?= $(GOPATH)/bin -GOBIN ?= /work/out/bin - LOCAL_ARCH := $(shell uname -m) ifeq ($(LOCAL_ARCH),x86_64) -GOARCH_LOCAL := amd64 + TARGET_ARCH ?= amd64 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8) -GOARCH_LOCAL := arm64 + TARGET_ARCH ?= arm64 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv) -GOARCH_LOCAL := arm + TARGET_ARCH ?= arm else -GOARCH_LOCAL := $(LOCAL_ARCH) + $(error "This system's architecture $(LOCAL_ARCH) isn't recognized/supported") endif -GOARCH ?= $(GOARCH_LOCAL) - LOCAL_OS := $(shell uname) ifeq ($(LOCAL_OS),Linux) - GOOS_LOCAL = linux + TARGET_OS ?= linux + READLINK_FLAGS="-f" else ifeq ($(LOCAL_OS),Darwin) - GOOS_LOCAL = darwin + TARGET_OS ?= darwin + READLINK_FLAGS="" else $(error "This system's OS $(LOCAL_OS) isn't recognized/supported") endif -GOOS ?= $(GOOS_LOCAL) +REPO_ROOT = $(shell git rev-parse --show-toplevel) +REPO_NAME = $(shell basename $(REPO_ROOT)) +TARGET_OUT ?= $(HOME)/istio_out/$(REPO_NAME) + +ifeq ($(BUILD_WITH_CONTAINER),1) +CONTAINER_CLI ?= docker +DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock +IMG ?= gcr.io/istio-testing/build-tools:2019-09-23T12-48-14 +UID = $(shell id -u) +PWD = $(shell pwd) + +# Determine the timezone across various platforms to pass into the +# docker run operation. This operation assumes zoneinfo is within +# the path of the file. +TIMEZONE=`readlink $(READLINK_FLAGS) /etc/localtime | sed -e 's/^.*zoneinfo\///'` -RUN = docker run -t -i --sig-proxy=true -u $(UID) --rm \ - -e GOOS="$(GOOS)" \ - -e GOARCH="$(GOARCH)" \ - -e GOBIN="$(GOBIN)" \ +RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \ -e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \ + -e TZ="$(TIMEZONE)" \ + -e TARGET_ARCH="$(TARGET_ARCH)" \ + -e TARGET_OS="$(TARGET_OS)" \ -v /etc/passwd:/etc/passwd:ro \ - -v $(readlink /etc/localtime):/etc/localtime:ro \ - -v /var/run/docker.sock:/var/run/docker.sock \ + $(DOCKER_SOCKET_MOUNT) \ $(CONTAINER_OPTIONS) \ --mount type=bind,source="$(PWD)",destination="/work" \ - --mount type=volume,source=istio-go-mod,destination="/go/pkg/mod" \ - --mount type=volume,source=istio-go-cache,destination="/gocache" \ - --mount type=bind,source="$(GOBIN_SOURCE)",destination="/go/out/bin" \ + --mount type=bind,source="$(TARGET_OUT)",destination="/targetout" \ + --mount type=volume,source=home,destination="/home" \ -w /work $(IMG) else -export GOBIN ?= ./out/bin RUN = endif MAKE = $(RUN) make --no-print-directory -e -f Makefile.core.mk %: + @mkdir -p $(TARGET_OUT) @$(MAKE) $@ default: + @mkdir -p $(TARGET_OUT) @$(MAKE) .PHONY: default diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha index fe10e724e5cc..50586b88e27a 100644 --- a/common/.commonfiles.sha +++ b/common/.commonfiles.sha @@ -1 +1 @@ -c3b89849b7ad86801e4511bc915137981c1ec0f1 +b911e0db0ccb611ae4fd361d295af2ef5f8089d7 diff --git a/common/Makefile.common.mk b/common/Makefile.common.mk index b1f4727de3b1..ca32ee80d983 100644 --- a/common/Makefile.common.mk +++ b/common/Makefile.common.mk @@ -19,7 +19,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FINDFILES=find . \( -path ./vendor -o -path ./.git \) -prune -o -type f +FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./.github \) -prune -o -type f XARGS = xargs -0 -r lint-dockerfiles: @@ -36,17 +36,17 @@ lint-helm: lint-copyright-banner: @${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\ - ${XARGS} common/scripts/lint_copyright_banner.sh + ${XARGS} common/scripts/lint_copyright_banner.sh lint-go: - @${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | : | ${XARGS} golangci-lint run -j 8 -c ./common/config/.golangci.yml + @${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/lint_go.sh lint-python: - @${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d + @${FINDFILES} -name '*.py' \( ! \( -name '*_pb2.py' \) \) -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d lint-markdown: @${FINDFILES} -name '*.md' -print0 | ${XARGS} mdl --ignore-front-matter --style common/config/mdl.rb - @${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect + @${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect --white-list ${MARKDOWN_LINT_WHITELIST} lint-sass: @${FINDFILES} -name '*.scss' -print0 | ${XARGS} sass-lint -c common/config/sass-lint.yml --verbose @@ -54,7 +54,10 @@ lint-sass: lint-typescript: @${FINDFILES} -name '*.ts' -print0 | ${XARGS} tslint -c common/config/tslint.json -lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript +lint-protos: + @if test -d common-protos; then $(FINDFILES) -name '*.proto' -print0 | $(XARGS) -L 1 prototool lint --protoc-bin-path=/usr/bin/protoc --protoc-wkt-path=common-protos; fi + +lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript lint-protos format-go: @${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} goimports -w -local "istio.io" @@ -62,17 +65,21 @@ format-go: format-python: @${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --aggressive --aggressive -i +format-protos: + @$(FINDFILES) -name '*.proto' -print0 | $(XARGS) -L 1 prototool format -w + update-common: - @git clone --depth 1 --single-branch --branch master https://github.com/istio/common-files + @git clone -q --depth 1 --single-branch --branch master https://github.com/istio/common-files @cd common-files ; git rev-parse HEAD >files/common/.commonfiles.sha @rm -fr common - @cp -r common-files/files/* . + @cp -rT common-files/files . @rm -fr common-files update-common-protos: - @git clone --depth 1 --single-branch --branch master https://github.com/istio/common-files + @git clone -q --depth 1 --single-branch --branch master https://github.com/istio/common-files @cd common-files ; git rev-parse HEAD > common-protos/.commonfiles.sha + @rm -fr common-protos @cp -ar common-files/common-protos common-protos @rm -fr common-files -.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-pyhton lint-helm lint-markdown lint-sass lint-typescript lint-all format-go format-python update-common update-common-protos +.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-pyhton lint-helm lint-markdown lint-sass lint-typescript lint-protos lint-all format-go format-python format-protos update-common update-common-protos diff --git a/common/config/.golangci.yml b/common/config/.golangci.yml index 22649af4b362..2b4bc6c08c96 100644 --- a/common/config/.golangci.yml +++ b/common/config/.golangci.yml @@ -7,7 +7,7 @@ service: # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo. - golangci-lint-version: 1.16.x # use the fixed version to not introduce new linters unexpectedly + golangci-lint-version: 1.18.x # use the fixed version to not introduce new linters unexpectedly run: # timeout for analysis, e.g. 30s, 5m, default is 1m deadline: 20m @@ -42,6 +42,8 @@ linters: - nakedret - prealloc - scopelint + - funlen + - bodyclose fast: false linters-settings: @@ -74,6 +76,8 @@ linters-settings: # Default is to use a neutral variety of English. # Setting locale to US will correct the British spelling of 'colour' to 'color'. locale: US + ignore-words: + - cancelled lll: # max line length, lines longer will be reported. Default is 120. # '\t' is counted as 1 character by default, and can be changed with the tab-width option @@ -142,9 +146,9 @@ linters-settings: - unslice - valSwap - weakCond - - yodaStyleExpr # Unused + # - yodaStyleExpr # - appendAssign # - commentFormatting # - emptyStringTest diff --git a/common/config/mdl.rb b/common/config/mdl.rb index e3e58e962168..8764f94d7263 100644 --- a/common/config/mdl.rb +++ b/common/config/mdl.rb @@ -1,5 +1,5 @@ all -rule 'MD002', :level => 2 +rule 'MD002', :level => 1 rule 'MD007', :indent => 4 rule 'MD013', :line_length => 160, :code_blocks => false, :tables => false rule 'MD026', :punctuation => ".,;:!" diff --git a/common/scripts/gobuild.sh b/common/scripts/gobuild.sh index 52330f99a48c..55816731b13f 100755 --- a/common/scripts/gobuild.sh +++ b/common/scripts/gobuild.sh @@ -1,5 +1,12 @@ #!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY # +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + # Copyright Istio Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,16 +33,17 @@ fi SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" OUT=${1:?"output path"} -BUILDPATH=${2:?"path to build"} +shift set -e BUILD_GOOS=${GOOS:-linux} BUILD_GOARCH=${GOARCH:-amd64} GOBINARY=${GOBINARY:-go} +GOPKG="$GOPATH/pkg" BUILDINFO=${BUILDINFO:-""} STATIC=${STATIC:-1} -LDFLAGS="-extldflags -static" +LDFLAGS=${LDFLAGS:--extldflags -static} GOBUILDFLAGS=${GOBUILDFLAGS:-""} # Split GOBUILDFLAGS by spaces into an array called GOBUILDFLAGS_ARRAY. IFS=' ' read -r -a GOBUILDFLAGS_ARRAY <<< "$GOBUILDFLAGS" @@ -57,6 +65,7 @@ fi # BUILD LD_EXTRAFLAGS LD_EXTRAFLAGS="" + while read -r line; do LD_EXTRAFLAGS="${LD_EXTRAFLAGS} -X ${line}" done < "${BUILDINFO}" @@ -64,4 +73,6 @@ done < "${BUILDINFO}" time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \ ${V} "${GOBUILDFLAGS_ARRAY[@]}" ${GCFLAGS:+-gcflags "${GCFLAGS}"} \ -o "${OUT}" \ - -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${BUILDPATH}" + -trimpath \ + -pkgdir="${GOPKG}/${BUILD_GOOS}_${BUILD_GOARCH}" \ + -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${@}" diff --git a/common/scripts/lint_go.sh b/common/scripts/lint_go.sh new file mode 100755 index 000000000000..2f74e8a2960f --- /dev/null +++ b/common/scripts/lint_go.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +GOGC=25 golangci-lint run -c ./common/config/.golangci.yml diff --git a/common/scripts/report_build_info.sh b/common/scripts/report_build_info.sh index 16434a1d4878..06f9afcf5e9e 100755 --- a/common/scripts/report_build_info.sh +++ b/common/scripts/report_build_info.sh @@ -1,5 +1,12 @@ #!/bin/bash +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + # Copyright Istio Authors # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,11 +22,11 @@ # limitations under the License. if BUILD_GIT_REVISION=$(git rev-parse HEAD 2> /dev/null); then - if ! git diff-index --quiet HEAD; then - BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty" - fi + if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then + BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty" + fi else - BUILD_GIT_REVISION=unknown + BUILD_GIT_REVISION=unknown fi # Check for local changes @@ -29,26 +36,16 @@ else tree_status="Modified" fi -# XXX This needs to be updated to accommodate tags added after building, rather than prior to builds -RELEASE_TAG=$(git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --exact-match 2> /dev/null || echo "") - # security wanted VERSION='unknown' VERSION="${BUILD_GIT_REVISION}" -if [[ -n "${RELEASE_TAG}" ]]; then - VERSION="${RELEASE_TAG}" -elif [[ -n ${ISTIO_VERSION} ]]; then +if [[ -n ${ISTIO_VERSION} ]]; then VERSION="${ISTIO_VERSION}" fi -DOCKER_HUB="docker.io/istio" -if [[ -n ${ISTIO_DOCKER_HUB} ]]; then - DOCKER_HUB="${ISTIO_DOCKER_HUB}" -fi +GIT_DESCRIBE_TAG=$(git describe --tags) # used by common/scripts/gobuild.sh echo "istio.io/pkg/version.buildVersion=${VERSION}" echo "istio.io/pkg/version.buildGitRevision=${BUILD_GIT_REVISION}" -echo "istio.io/pkg/version.buildUser=$(whoami)" -echo "istio.io/pkg/version.buildHost=$(hostname -f)" -echo "istio.io/pkg/version.buildDockerHub=${DOCKER_HUB}" echo "istio.io/pkg/version.buildStatus=${tree_status}" +echo "istio.io/pkg/version.buildTag=${GIT_DESCRIBE_TAG}" diff --git a/mdl.rb b/mdl.rb new file mode 100644 index 000000000000..e3e58e962168 --- /dev/null +++ b/mdl.rb @@ -0,0 +1,12 @@ +all +rule 'MD002', :level => 2 +rule 'MD007', :indent => 4 +rule 'MD013', :line_length => 160, :code_blocks => false, :tables => false +rule 'MD026', :punctuation => ".,;:!" +exclude_rule 'MD013' +exclude_rule 'MD014' +exclude_rule 'MD030' +exclude_rule 'MD032' +exclude_rule 'MD033' +exclude_rule 'MD041' +exclude_rule 'MD046' diff --git a/scripts/lint_site.sh b/scripts/lint_site.sh index 46c690a789ff..afb786a2e781 100755 --- a/scripts/lint_site.sh +++ b/scripts/lint_site.sh @@ -39,7 +39,7 @@ check_content() { # create a throwaway copy of the content cp -R "${DIR}" "${TMP}" cp .spelling "${TMP}" - cp common/config/mdl.rb "${TMP}" + cp mdl.rb "${TMP}" # replace the {{< text >}} shortcodes with ```plain find "${TMP}" -type f -name \*.md -exec sed -E -i "s/\\{\\{< text .*>\}\}/\`\`\`plain/g" {} ";"