Update common files. (#5052)

.gitattributes

@@ -0,0 +1,9 @@
+*.descriptor      linguist-generated=true
+*.descriptor      -diff -merge
+*.descriptor_set  linguist-generated=true
+*.descriptor_set  -diff -merge
+*.pb.html linguist-generated=true
+*.pb.go linguist-generated=true
+*.gen.go linguist-generated=true
+*.gen.yaml linguist-generated=true
+*_pb2.py linguist-generated=true

Makefile

@@ -28,62 +28,68 @@
 # figure out all the tools you need in your environment to make that work.
 export BUILD_WITH_CONTAINER ?= 0
 
-ifeq ($(BUILD_WITH_CONTAINER),1)
-IMG = gcr.io/istio-testing/build-tools:2019-08-29T13-57-48
-UID = $(shell id -u)
-PWD = $(shell pwd)
-GOBIN_SOURCE ?= $(GOPATH)/bin
-GOBIN ?= /work/out/bin
-
 LOCAL_ARCH := $(shell uname -m)
 ifeq ($(LOCAL_ARCH),x86_64)
-GOARCH_LOCAL := amd64
+    TARGET_ARCH ?= amd64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 5),armv8)
-GOARCH_LOCAL := arm64
+    TARGET_ARCH ?= arm64
 else ifeq ($(shell echo $(LOCAL_ARCH) | head -c 4),armv)
-GOARCH_LOCAL := arm
+    TARGET_ARCH ?= arm
 else
-GOARCH_LOCAL := $(LOCAL_ARCH)
+   $(error "This system's architecture $(LOCAL_ARCH) isn't recognized/supported")
 endif
 
-GOARCH ?= $(GOARCH_LOCAL)
-
 LOCAL_OS := $(shell uname)
 ifeq ($(LOCAL_OS),Linux)
-   GOOS_LOCAL = linux
+   TARGET_OS ?= linux
+   READLINK_FLAGS="-f"
 else ifeq ($(LOCAL_OS),Darwin)
-   GOOS_LOCAL = darwin
+   TARGET_OS ?= darwin
+   READLINK_FLAGS=""
 else
    $(error "This system's OS $(LOCAL_OS) isn't recognized/supported")
 endif
 
-GOOS ?= $(GOOS_LOCAL)
+REPO_ROOT = $(shell git rev-parse --show-toplevel)
+REPO_NAME = $(shell basename $(REPO_ROOT))
+TARGET_OUT ?= $(HOME)/istio_out/$(REPO_NAME)
+
+ifeq ($(BUILD_WITH_CONTAINER),1)
+CONTAINER_CLI ?= docker
+DOCKER_SOCKET_MOUNT ?= -v /var/run/docker.sock:/var/run/docker.sock
+IMG ?= gcr.io/istio-testing/build-tools:2019-09-23T12-48-14
+UID = $(shell id -u)
+PWD = $(shell pwd)
+
+# Determine the timezone across various platforms to pass into the
+# docker run operation. This operation assumes zoneinfo is within
+# the path of the file.
+TIMEZONE=`readlink $(READLINK_FLAGS) /etc/localtime | sed -e 's/^.*zoneinfo\///'`
 
-RUN = docker run -t -i --sig-proxy=true -u $(UID) --rm \
-	-e GOOS="$(GOOS)" \
-	-e GOARCH="$(GOARCH)" \
-	-e GOBIN="$(GOBIN)" \
+RUN = $(CONTAINER_CLI) run -t -i --sig-proxy=true -u $(UID) --rm \
 	-e BUILD_WITH_CONTAINER="$(BUILD_WITH_CONTAINER)" \
+	-e TZ="$(TIMEZONE)" \
+	-e TARGET_ARCH="$(TARGET_ARCH)" \
+	-e TARGET_OS="$(TARGET_OS)" \
 	-v /etc/passwd:/etc/passwd:ro \
-	-v $(readlink /etc/localtime):/etc/localtime:ro \
-	-v /var/run/docker.sock:/var/run/docker.sock \
+	$(DOCKER_SOCKET_MOUNT) \
 	$(CONTAINER_OPTIONS) \
 	--mount type=bind,source="$(PWD)",destination="/work" \
-	--mount type=volume,source=istio-go-mod,destination="/go/pkg/mod" \
-	--mount type=volume,source=istio-go-cache,destination="/gocache" \
-	--mount type=bind,source="$(GOBIN_SOURCE)",destination="/go/out/bin" \
+	--mount type=bind,source="$(TARGET_OUT)",destination="/targetout" \
+	--mount type=volume,source=home,destination="/home" \
 	-w /work $(IMG)
 else
-export GOBIN ?= ./out/bin
 RUN =
 endif
 
 MAKE = $(RUN) make --no-print-directory -e -f Makefile.core.mk
 
 %:
+	@mkdir -p $(TARGET_OUT)
 	@$(MAKE) $@
 
 default:
+	@mkdir -p $(TARGET_OUT)
 	@$(MAKE)
 
 .PHONY: default

common/.commonfiles.sha

@@ -1 +1 @@
-c3b89849b7ad86801e4511bc915137981c1ec0f1
+b911e0db0ccb611ae4fd361d295af2ef5f8089d7

common/Makefile.common.mk

@@ -19,7 +19,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FINDFILES=find . \( -path ./vendor -o -path ./.git \) -prune -o -type f
+FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./.github \) -prune -o -type f
 XARGS = xargs -0 -r
 
 lint-dockerfiles:
@@ -36,43 +36,50 @@ lint-helm:
 
 lint-copyright-banner:
 	@${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\
-	 	${XARGS} common/scripts/lint_copyright_banner.sh
+		${XARGS} common/scripts/lint_copyright_banner.sh
 
 lint-go:
-	@${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | : | ${XARGS} golangci-lint run -j 8 -c ./common/config/.golangci.yml
+	@${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/lint_go.sh
 
 lint-python:
-	@${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d
+	@${FINDFILES} -name '*.py' \( ! \( -name '*_pb2.py' \) \) -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d
 
 lint-markdown:
 	@${FINDFILES} -name '*.md' -print0 | ${XARGS} mdl --ignore-front-matter --style common/config/mdl.rb
-	@${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect
+	@${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect --white-list ${MARKDOWN_LINT_WHITELIST}
 
 lint-sass:
 	@${FINDFILES} -name '*.scss' -print0 | ${XARGS} sass-lint -c common/config/sass-lint.yml --verbose
 
 lint-typescript:
 	@${FINDFILES} -name '*.ts' -print0 | ${XARGS} tslint -c common/config/tslint.json
 
-lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript
+lint-protos:
+	@if test -d common-protos; then $(FINDFILES) -name '*.proto' -print0 | $(XARGS) -L 1 prototool lint --protoc-bin-path=/usr/bin/protoc --protoc-wkt-path=common-protos; fi
+
+lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript lint-protos
 
 format-go:
 	@${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} goimports -w -local "istio.io"
 
 format-python:
 	@${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --aggressive --aggressive -i
 
+format-protos:
+	@$(FINDFILES) -name '*.proto' -print0 | $(XARGS) -L 1 prototool format -w
+
 update-common:
-	@git clone --depth 1 --single-branch --branch master https://github.com/istio/common-files
+	@git clone -q --depth 1 --single-branch --branch master https://github.com/istio/common-files
 	@cd common-files ; git rev-parse HEAD >files/common/.commonfiles.sha
 	@rm -fr common
-	@cp -r common-files/files/* .
+	@cp -rT common-files/files .
 	@rm -fr common-files
 
 update-common-protos:
-	@git clone --depth 1 --single-branch --branch master https://github.com/istio/common-files
+	@git clone -q --depth 1 --single-branch --branch master https://github.com/istio/common-files
 	@cd common-files ; git rev-parse HEAD > common-protos/.commonfiles.sha
+	@rm -fr common-protos
 	@cp -ar common-files/common-protos common-protos
 	@rm -fr common-files
 
-.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-pyhton lint-helm lint-markdown lint-sass lint-typescript lint-all format-go format-python update-common update-common-protos
+.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-pyhton lint-helm lint-markdown lint-sass lint-typescript lint-protos lint-all format-go format-python format-protos update-common update-common-protos

common/config/.golangci.yml

@@ -7,7 +7,7 @@
 
 service:
   # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo.
-  golangci-lint-version: 1.16.x # use the fixed version to not introduce new linters unexpectedly
+  golangci-lint-version: 1.18.x # use the fixed version to not introduce new linters unexpectedly
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   deadline: 20m
@@ -42,6 +42,8 @@ linters:
     - nakedret
     - prealloc
     - scopelint
+    - funlen
+    - bodyclose
   fast: false
 
 linters-settings:
@@ -74,6 +76,8 @@ linters-settings:
     # Default is to use a neutral variety of English.
     # Setting locale to US will correct the British spelling of 'colour' to 'color'.
     locale: US
+    ignore-words:
+    - cancelled
   lll:
     # max line length, lines longer will be reported. Default is 120.
     # '\t' is counted as 1 character by default, and can be changed with the tab-width option
@@ -142,9 +146,9 @@ linters-settings:
       - unslice
       - valSwap
       - weakCond
-      - yodaStyleExpr
 
       # Unused
+      # - yodaStyleExpr
       # - appendAssign
       # - commentFormatting
       # - emptyStringTest

common/config/mdl.rb

@@ -1,5 +1,5 @@
 all
-rule 'MD002', :level => 2
+rule 'MD002', :level => 1
 rule 'MD007', :indent => 4
 rule 'MD013', :line_length => 160, :code_blocks => false, :tables => false
 rule 'MD026', :punctuation => ".,;:!"

common/scripts/gobuild.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
+
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
 #
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
 # Copyright Istio Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -26,16 +33,17 @@ fi
 SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 OUT=${1:?"output path"}
-BUILDPATH=${2:?"path to build"}
+shift
 
 set -e
 
 BUILD_GOOS=${GOOS:-linux}
 BUILD_GOARCH=${GOARCH:-amd64}
 GOBINARY=${GOBINARY:-go}
+GOPKG="$GOPATH/pkg"
 BUILDINFO=${BUILDINFO:-""}
 STATIC=${STATIC:-1}
-LDFLAGS="-extldflags -static"
+LDFLAGS=${LDFLAGS:--extldflags -static}
 GOBUILDFLAGS=${GOBUILDFLAGS:-""}
 # Split GOBUILDFLAGS by spaces into an array called GOBUILDFLAGS_ARRAY.
 IFS=' ' read -r -a GOBUILDFLAGS_ARRAY <<< "$GOBUILDFLAGS"
@@ -57,11 +65,14 @@ fi
 
 # BUILD LD_EXTRAFLAGS
 LD_EXTRAFLAGS=""
+
 while read -r line; do
     LD_EXTRAFLAGS="${LD_EXTRAFLAGS} -X ${line}"
 done < "${BUILDINFO}"
 
 time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \
         ${V} "${GOBUILDFLAGS_ARRAY[@]}" ${GCFLAGS:+-gcflags "${GCFLAGS}"} \
         -o "${OUT}" \
-        -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${BUILDPATH}"
+        -trimpath \
+        -pkgdir="${GOPKG}/${BUILD_GOOS}_${BUILD_GOARCH}" \
+        -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${@}"

common/scripts/lint_go.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+GOGC=25 golangci-lint run -c ./common/config/.golangci.yml

common/scripts/report_build_info.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
 # Copyright Istio Authors
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,11 +22,11 @@
 #   limitations under the License.
 
 if BUILD_GIT_REVISION=$(git rev-parse HEAD 2> /dev/null); then
-    if ! git diff-index --quiet HEAD; then
-        BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty"
-    fi
+  if [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then
+    BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty"
+  fi
 else
-    BUILD_GIT_REVISION=unknown
+  BUILD_GIT_REVISION=unknown
 fi
 
 # Check for local changes
@@ -29,26 +36,16 @@ else
   tree_status="Modified"
 fi
 
-# XXX This needs to be updated to accommodate tags added after building, rather than prior to builds
-RELEASE_TAG=$(git describe --match '[0-9]*\.[0-9]*\.[0-9]*' --exact-match 2> /dev/null || echo "")
-
 # security wanted VERSION='unknown'
 VERSION="${BUILD_GIT_REVISION}"
-if [[ -n "${RELEASE_TAG}" ]]; then
-  VERSION="${RELEASE_TAG}"
-elif [[ -n ${ISTIO_VERSION} ]]; then
+if [[ -n ${ISTIO_VERSION} ]]; then
   VERSION="${ISTIO_VERSION}"
 fi
 
-DOCKER_HUB="docker.io/istio"
-if [[ -n ${ISTIO_DOCKER_HUB} ]]; then
-  DOCKER_HUB="${ISTIO_DOCKER_HUB}"
-fi
+GIT_DESCRIBE_TAG=$(git describe --tags)
 
 # used by common/scripts/gobuild.sh
 echo "istio.io/pkg/version.buildVersion=${VERSION}"
 echo "istio.io/pkg/version.buildGitRevision=${BUILD_GIT_REVISION}"
-echo "istio.io/pkg/version.buildUser=$(whoami)"
-echo "istio.io/pkg/version.buildHost=$(hostname -f)"
-echo "istio.io/pkg/version.buildDockerHub=${DOCKER_HUB}"
 echo "istio.io/pkg/version.buildStatus=${tree_status}"
+echo "istio.io/pkg/version.buildTag=${GIT_DESCRIBE_TAG}"

mdl.rb

@@ -0,0 +1,12 @@
+all
+rule 'MD002', :level => 2
+rule 'MD007', :indent => 4
+rule 'MD013', :line_length => 160, :code_blocks => false, :tables => false
+rule 'MD026', :punctuation => ".,;:!"
+exclude_rule 'MD013'
+exclude_rule 'MD014'
+exclude_rule 'MD030'
+exclude_rule 'MD032'
+exclude_rule 'MD033'
+exclude_rule 'MD041'
+exclude_rule 'MD046'

scripts/lint_site.sh

@@ -39,7 +39,7 @@ check_content() {
     # create a throwaway copy of the content
     cp -R "${DIR}" "${TMP}"
     cp .spelling "${TMP}"
-    cp common/config/mdl.rb "${TMP}"
+    cp mdl.rb "${TMP}"
 
     # replace the {{< text >}} shortcodes with ```plain
     find "${TMP}" -type f -name \*.md -exec sed -E -i "s/\\{\\{< text .*>\}\}/\`\`\`plain/g" {} ";"