From f08769f55b4b0fdb68b5c6882e46cf786661f4e3 Mon Sep 17 00:00:00 2001 From: Jonh Wendell Date: Mon, 9 Oct 2023 09:49:24 -0400 Subject: [PATCH] OSSM-4956: Use upstream build container and common-files --- .devcontainer/devcontainer.json | 33 ++ .gitattributes | 15 + BUGS-AND-FEATURE-REQUESTS.md | 10 + CONTRIBUTING.md | 5 + LICENSE | 8 +- Makefile | 512 +++--------------------- Makefile.core.mk | 432 ++++++++++++++++++++ Makefile.overrides.mk | 13 + SUPPORT.md | 7 + common/.commonfiles.sha | 1 + common/Makefile.common.mk | 121 ++++++ common/config/.golangci-format.yml | 59 +++ common/config/.golangci.yml | 331 +++++++++++++++ common/config/.hadolint.yml | 15 + common/config/.yamllint.yml | 29 ++ common/config/license-lint.yml | 128 ++++++ common/config/mdl.rb | 12 + common/config/sass-lint.yml | 98 +++++ common/config/tslint.json | 25 ++ common/scripts/check_clean_repo.sh | 50 +++ common/scripts/copyright-banner-go.txt | 14 + common/scripts/fix_copyright_banner.sh | 39 ++ common/scripts/format_go.sh | 24 ++ common/scripts/gobuild.sh | 83 ++++ common/scripts/kind_provisioner.sh | 438 ++++++++++++++++++++ common/scripts/lint_copyright_banner.sh | 39 ++ common/scripts/lint_go.sh | 28 ++ common/scripts/metallb.yaml | 409 +++++++++++++++++++ common/scripts/report_build_info.sh | 48 +++ common/scripts/run.sh | 62 +++ common/scripts/setup_env.sh | 233 +++++++++++ common/scripts/tracing.sh | 118 ++++++ out/.env | 22 + tools/commonfiles-postprocess.sh | 19 + tools/golangci-override.yaml | 10 + 35 files changed, 3024 insertions(+), 466 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .gitattributes create mode 100644 BUGS-AND-FEATURE-REQUESTS.md create mode 100644 CONTRIBUTING.md create mode 100644 Makefile.core.mk create mode 100644 Makefile.overrides.mk create mode 100644 SUPPORT.md create mode 100644 common/.commonfiles.sha create mode 100644 common/Makefile.common.mk create mode 100644 common/config/.golangci-format.yml create mode 100644 common/config/.golangci.yml create mode 100644 common/config/.hadolint.yml create mode 100644 common/config/.yamllint.yml create mode 100644 common/config/license-lint.yml create mode 100644 common/config/mdl.rb create mode 100644 common/config/sass-lint.yml create mode 100644 common/config/tslint.json create mode 100755 common/scripts/check_clean_repo.sh create mode 100644 common/scripts/copyright-banner-go.txt create mode 100755 common/scripts/fix_copyright_banner.sh create mode 100755 common/scripts/format_go.sh create mode 100755 common/scripts/gobuild.sh create mode 100644 common/scripts/kind_provisioner.sh create mode 100755 common/scripts/lint_copyright_banner.sh create mode 100755 common/scripts/lint_go.sh create mode 100644 common/scripts/metallb.yaml create mode 100755 common/scripts/report_build_info.sh create mode 100755 common/scripts/run.sh create mode 100755 common/scripts/setup_env.sh create mode 100755 common/scripts/tracing.sh create mode 100644 out/.env create mode 100755 tools/commonfiles-postprocess.sh create mode 100644 tools/golangci-override.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..d73a7554e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,33 @@ +{ + "name": "istio build-tools", + "image": "gcr.io/istio-testing/build-tools:master-d5b39ac8e423b269748c9c1e9692a853db960fc9", + "privileged": true, + "remoteEnv": { + "USE_GKE_GCLOUD_AUTH_PLUGIN": "True", + "BUILD_WITH_CONTAINER": "0", + "CARGO_HOME": "/home/.cargo", + "RUSTUP_HOME": "/home/.rustup" + }, + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/mpriscella/features/kind:1": {} + }, + "customizations": { + "vscode": { + "extensions": [ + "golang.go", + "rust-lang.rust-analyzer", + "eamodio.gitlens", + "zxh404.vscode-proto3", + "ms-azuretools.vscode-docker", + "redhat.vscode-yaml", + "IBM.output-colorizer" + ], + "settings": { + "files.eol": "\n", + "go.useLanguageServer": true, + "go.lintTool": "golangci-lint" + } + } + } +} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..86305d16e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,15 @@ +*.descriptor linguist-generated=true +*.descriptor -diff -merge +*.descriptor_set linguist-generated=true +*.descriptor_set -diff -merge +*.pb.html linguist-generated=true +*.pb.go linguist-generated=true +*.gen.go linguist-generated=true +*.gen.yaml linguist-generated=true +*.gen.json linguist-generated=true +*_pb2.py linguist-generated=true +go.sum merge=union +vendor/** linguist-vendored +common/** linguist-vendored +archive/** linquist-vendored +**/vmlinux.h linquist-vendored diff --git a/BUGS-AND-FEATURE-REQUESTS.md b/BUGS-AND-FEATURE-REQUESTS.md new file mode 100644 index 000000000..7b97d0e4d --- /dev/null +++ b/BUGS-AND-FEATURE-REQUESTS.md @@ -0,0 +1,10 @@ +# Bugs and Feature Requests + +You can report bugs and feature requests to the Istio team in one of three places: + +- [Product Bugs and Feature Requests](https://github.com/istio/istio/issues) +- [Documentation Bugs and Feature Requests](https://github.com/istio/istio.io/issues) +- [Community and Governance Issues](https://github.com/istio/community/issues) + +For security vulnerabilities, please don't report a bug (which is public) and instead follow +[these procedures](https://istio.io/about/security-vulnerabilities/). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..7b55cf47a --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,5 @@ +# Contribution guidelines + +So you want to hack on Istio? Yay! Please refer to Istio's overall +[contribution guidelines](https://github.com/istio/community/blob/master/CONTRIBUTING.md) +to find out how you can help. diff --git a/LICENSE b/LICENSE index b085857df..bb7b19dec 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,5 @@ - Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -178,7 +179,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,8 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2020 Red Hat Inc - + Copyright 2016-2022 Istio Authors Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/Makefile b/Makefile index 8e7217df9..402171d27 100644 --- a/Makefile +++ b/Makefile @@ -1,481 +1,69 @@ -## Copyright 2019 Red Hat, Inc. -## -## Licensed under the Apache License, Version 2.0 (the "License"); -## you may not use this file except in compliance with the License. -## You may obtain a copy of the License at -## -## http://www.apache.org/licenses/LICENSE-2.0 -## -## Unless required by applicable law or agreed to in writing, software -## distributed under the License is distributed on an "AS IS" BASIS, -## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -## See the License for the specific language governing permissions and -## limitations under the License. - --include Makefile.overrides - -# VERSION defines the project version for the bundle. -# Update this value when you upgrade the version of your project. -# To re-generate a bundle for another specific version without changing the standard setup, you can: -# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) -# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) -VERSION ?= 3.0.0 -MINOR_VERSION := $(shell v='$(VERSION)'; echo "$${v%.*}") -MINIMUM_SUPPORTED_VERSION ?= v3.0 - -OPERATOR_NAME ?= sailoperator - -# Istio repository to pull charts from -ISTIO_REPOSITORY ?= maistra/istio -# the branch to use when updating the commit hash below -ISTIO_BRANCH_30 ?= maistra-3.0 -# the current commit of ${ISTIO_REPOSITORY} for v3.0. This will be used to copy charts -ISTIO_COMMIT_30 ?= ffdc450902e8fb7c2cd02eeca15d38d65b03f14b - -# Istio images names -ISTIO_CNI_IMAGE_NAME ?= install-cni -ISTIO_PILOT_IMAGE_NAME ?= pilot -ISTIO_PROXY_IMAGE_NAME ?= proxyv2 - -# GitHub creds -GITHUB_USER ?= maistra-bot -GITHUB_TOKEN ?= - -SOURCE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) - -# Git repository state -ifndef GIT_TAG -GIT_TAG := $(shell git describe 2> /dev/null || echo "unknown") -endif -ifndef GIT_REVISION -GIT_REVISION := $(shell git rev-parse --verify HEAD 2> /dev/null || echo "unknown") -endif -ifndef GIT_STATUS -GIT_STATUS := $(shell git diff-index --quiet HEAD -- 2> /dev/null; if [ "$$?" = "0" ]; then echo Clean; else echo Modified; fi) -endif - -# Linker flags for the go builds -GO_MODULE = maistra.io/istio-operator -LD_EXTRAFLAGS = -X ${GO_MODULE}/pkg/version.buildVersion=${VERSION} -LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildGitRevision=${GIT_REVISION} -LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildTag=${GIT_TAG} -LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildStatus=${GIT_STATUS} -LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.minimumSupportedVersion=${MINIMUM_SUPPORTED_VERSION} -LD_FLAGS = -extldflags -static ${LD_EXTRAFLAGS} -s -w - -# Image hub to use -HUB ?= quay.io/maistra-dev -# Image tag to use -TAG ?= ${MINOR_VERSION}-latest -# Image base to use -IMAGE_BASE ?= istio-ubi9-operator -# Image URL to use all building/pushing image targets -IMAGE ?= ${HUB}/${IMAGE_BASE}:${TAG} -# Namespace to deploy the controller in -NAMESPACE ?= istio-operator -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.26.0 - -# CHANNELS define the bundle channels used in the bundle. -# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") -# To re-generate a bundle for other specific channels without changing the standard setup, you can: -# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) -# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") -CHANNELS ?= "3.0" -ifneq ($(origin CHANNELS), undefined) -BUNDLE_CHANNELS := --channels=\"$(CHANNELS)\" -endif - -# DEFAULT_CHANNEL defines the default channel used in the bundle. -# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") -# To re-generate a bundle for any other default channel without changing the default setup, you can: -# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) -# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") -ifneq ($(origin DEFAULT_CHANNEL), undefined) -BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) -endif -BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) - -# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. -# This variable is used to construct full image tags for bundle and catalog images. +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY # -# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both -# quay.io/maistra-dev/istio-ubi9-operator-bundle:$VERSION and quay.io/maistra-dev/istio-ubi9-operator-catalog:$VERSION. -IMAGE_TAG_BASE ?= ${HUB}/${IMAGE_BASE} - -BUNDLE_MANIFEST_DATE := $(shell cat bundle/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml 2>/dev/null | grep createdAt | awk '{print $$2}') - -# BUNDLE_IMG defines the image:tag used for the bundle. -# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) -BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) - -# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command -BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) - -# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests -# You can enable this value if you would like to use SHA Based Digests -# To enable set flag to true -USE_IMAGE_DIGESTS ?= false -ifeq ($(USE_IMAGE_DIGESTS), true) - BUNDLE_GEN_FLAGS += --use-image-digests -endif - -TODAY ?= $(shell date -I) - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk commands is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Testing - -.PHONY: test -test: envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out - -.PHONY: test.scorecard ## Runs the operator scorecard test. Needs a valid k8s cluster as pointed by the KUBECONFIG variable -test.scorecard: operator-sdk - $(OPERATOR_SDK) scorecard bundle - -##@ Build - -.PHONY: build -build: ## Build manager binary. - CGO_ENABLED=0 go build -o bin/manager -ldflags '${LD_FLAGS}' main.go - -.PHONY: run -run: gen ## Run a controller from your host. - POD_NAMESPACE=${NAMESPACE} go run ./main.go --config-file=./hack/config.properties --resource-directory=./resources - -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - docker build -t ${IMAGE} --build-arg GIT_TAG=${GIT_TAG} --build-arg GIT_REVISION=${GIT_REVISION} --build-arg GIT_STATUS=${GIT_STATUS} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - docker push ${IMAGE} - -.PHONY: docker-push-nightly ## Build and push nightly docker image with the manager. -docker-push-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) -docker-push-nightly: docker-build - docker push ${IMAGE} - docker tag ${IMAGE} $(HUB)/istio-ubi9-operator:$(MINOR_VERSION)-latest - docker push $(HUB)/istio-ubi9-operator:$(MINOR_VERSION)-latest - -# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple -# architectures. (i.e. make docker-buildx IMAGE=myregistry/mypoperator:0.0.1). To use this option you need to: -# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ -# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMAGE=> then the export will fail) -# To properly provided solutions that supports more than one platform you should use this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: test ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - docker buildx create --name project-v3-builder - docker buildx use project-v3-builder - - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMAGE} -f Dockerfile.cross . - - docker buildx rm project-v3-builder - rm Dockerfile.cross - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". -.PHONY: install -install: gen-manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - kubectl create ns ${NAMESPACE} || echo "namespace ${NAMESPACE} already exists" - $(KUSTOMIZE) build config/crd | kubectl apply -f - - -.PHONY: uninstall -uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - $(info NAMESPACE: $(NAMESPACE)) - $(MAKE) -s deploy-yaml | kubectl apply -f - - -.PHONY: deploy-yaml -deploy-yaml: kustomize ## Outputs YAML manifests needed to deploy the controller - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMAGE} - cd config/default && $(KUSTOMIZE) edit set namespace ${NAMESPACE} - $(KUSTOMIZE) build config/default - -.PHONY: deploy-olm -deploy-olm: bundle bundle-build bundle-push ## Builds and pushes the operator OLM bundle and then deploys the operator using OLM - kubectl create ns ${NAMESPACE} || echo "namespace ${NAMESPACE} already exists" - $(OPERATOR_SDK) run bundle $(BUNDLE_IMG) -n ${NAMESPACE} - -.PHONY: undeploy -undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(MAKE) deploy-yaml | kubectl delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: undeploy-olm -undeploy-olm: operator-sdk ## Undeploys the operator from the cluster (used only if operator was installed via OLM) - $(OPERATOR_SDK) cleanup sailoperator --delete-all -n ${NAMESPACE} - -.PHONY: deploy-example -deploy-example: deploy-example-openshift - -.PHONY: deploy-example-openshift -deploy-example-openshift: ## Deploy an example Istio resource on OpenShift - kubectl create ns istio-system || echo "namespace istio-system already exists" - kubectl apply -n istio-system -f config/samples/istio-sample-openshift.yaml - -.PHONY: deploy-example-kubernetes -deploy-example-kubernetes: ## Deploy an example Istio resource on Kubernetes - kubectl create ns istio-system || echo "namespace istio-system already exists" - kubectl apply -n istio-system -f config/samples/istio-sample-kubernetes.yaml - -##@ Generated Code & Resources - -.PHONY: gen-manifests -gen-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - -.PHONY: gen-code -gen-code: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: gen-charts -gen-charts: ## Pull charts from maistra/istio repository - hack/download-charts.sh v${MINOR_VERSION} https://github.com/${ISTIO_REPOSITORY} ${ISTIO_COMMIT_30} - -.PHONY: gen ## Generate everything -gen: controller-gen gen-manifests gen-code gen-charts bundle - -.PHONY: gen-check -gen-check: gen restore-manifest-dates check-clean-repo ## Verifies that changes in generated resources have been checked in - -.PHONY: restore-manifest-dates -restore-manifest-dates: -ifneq "${BUNDLE_MANIFEST_DATE}" "" - @sed -i -e "s/\(createdAt:\).*/\1 \"${BUNDLE_MANIFEST_DATE}\"/" bundle/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml -endif - -.PHONY: check-clean-repo -check-clean-repo: - @if [[ -n $$(git status --porcelain) ]]; then git status; git diff; echo "ERROR: Some files need to be updated, please run 'make gen' and include any changed files in your PR"; exit 1; fi - -.PHONY: update-istio -update-istio: ## Updates the Istio commit hash to latest on ${ISTIO_BRANCH_30} - $(eval ISTIO_COMMIT_30=$(shell git ls-remote https://github.com/${ISTIO_REPOSITORY}.git | grep ${ISTIO_BRANCH_30} | cut -f 1)) - @echo Updating to ${ISTIO_REPOSITORY}@${ISTIO_COMMIT_30}; sed -i -e "s/^\(ISTIO_COMMIT_30 ?= \).*$$/\1${ISTIO_COMMIT_30}/g" Makefile - -.PHONY: patch-istio-images -patch-istio-images: ## Patch the Istio images in the ClusterServiceVersion with the right tags - sed -i -e "s|images3_0.cni: .*|images3_0.cni: $(HUB)/$(ISTIO_CNI_IMAGE_NAME):$(TAG)|" \ - -e "s|images3_0.istiod: .*|images3_0.istiod: $(HUB)/$(ISTIO_PILOT_IMAGE_NAME):$(TAG)|" \ - -e "s|images3_0.proxy: .*|images3_0.proxy: $(HUB)/$(ISTIO_PROXY_IMAGE_NAME):$(TAG)|" \ - $(shell ls bundle/manifests/*.clusterserviceversion.yaml) - -##@ Build Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -## Tool Versions -OPERATOR_SDK_VERSION ?= v1.31.0 -KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.11.1 +SHELL := /bin/bash -KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. -$(KUSTOMIZE): $(LOCALBIN) - if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION) > /dev/stderr; then \ - echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing." > /dev/stderr; \ - rm -rf $(LOCALBIN)/kustomize; \ - fi - test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) > /dev/stderr; } +# allow optional per-repo overrides +-include Makefile.overrides.mk -.PHONY: operator-sdk -operator-sdk: $(OPERATOR_SDK) -operator-sdk: OS=$(shell go env GOOS) -operator-sdk: ARCH=$(shell go env GOARCH) -$(OPERATOR_SDK): $(LOCALBIN) - @if test -x $(LOCALBIN)/operator-sdk && ! $(LOCALBIN)/operator-sdk version | grep -q $(OPERATOR_SDK_VERSION); then \ - echo "$(LOCALBIN)/operator-sdk version is not expected $(OPERATOR_SDK_VERSION). Removing it before installing."; \ - rm -rf $(LOCALBIN)/operator-sdk; \ - fi - test -s $(LOCALBIN)/operator-sdk || \ - curl -sSLo $(LOCALBIN)/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(OS)_$(ARCH) && \ - chmod +x $(LOCALBIN)/operator-sdk; +# Set the environment variable BUILD_WITH_CONTAINER to use a container +# to build the repo. The only dependencies in this mode are to have make and +# docker. If you'd rather build with a local tool chain instead, you'll need to +# figure out all the tools you need in your environment to make that work. +export BUILD_WITH_CONTAINER ?= 0 -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. -$(CONTROLLER_GEN): $(LOCALBIN) - test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ - GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) +ifeq ($(BUILD_WITH_CONTAINER),1) -.PHONY: envtest -envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. -$(ENVTEST): $(LOCALBIN) - test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest +# An export free of arguments in a Makefile places all variables in the Makefile into the +# environment. This is needed to allow overrides from Makefile.overrides.mk. +export -.PHONY: bundle -bundle: gen kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. - $(OPERATOR_SDK) generate kustomize manifests -q - cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE) - sed -i "s|^\( containerImage:\).*$$|\1 ${IMAGE}|g" config/manifests/bases/${OPERATOR_NAME}.clusterserviceversion.yaml - $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) - $(OPERATOR_SDK) bundle validate ./bundle +RUN = ./common/scripts/run.sh -.PHONY: bundle-build -bundle-build: ## Build the bundle image. - docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . +MAKE_DOCKER = $(RUN) make --no-print-directory -e -f Makefile.core.mk -.PHONY: bundle-push -bundle-push: ## Push the bundle image. - $(MAKE) docker-push IMAGE=$(BUNDLE_IMG) +%: + @$(MAKE_DOCKER) $@ -.PHONY: bundle-publish -bundle-publish: patch-istio-images ## Create a PR for publishing in OperatorHub - export GIT_USER=$(GITHUB_USER); \ - export GITHUB_TOKEN=$(GITHUB_TOKEN); \ - export OPERATOR_VERSION=$(OPERATOR_VERSION); \ - export OPERATOR_NAME=$(OPERATOR_NAME); \ - ./hack/operatorhub/publish-bundle.sh +default: + @$(MAKE_DOCKER) -.PHONY: bundle-nightly ## Generate nightly bundle -bundle-nightly: - $(MAKE) bundle VERSION=${VERSION}-nightly-${TODAY} CHANNELS=$(MINOR_VERSION)-nightly TAG=$(MINOR_VERSION)-nightly-$(TODAY) +shell: + @$(RUN) /bin/bash -.PHONY: bundle-publish-nightly -bundle-publish-nightly: OPERATOR_VERSION=$(VERSION)-nightly-$(TODAY) -bundle-publish-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) -bundle-publish-nightly: bundle-nightly bundle-publish +.PHONY: default shell -.PHONY: opm -OPM = ./bin/opm -opm: ## Download opm locally if necessary. -ifeq (,$(wildcard $(OPM))) -ifeq (,$(shell which opm 2>/dev/null)) - @{ \ - set -e ;\ - mkdir -p $(dir $(OPM)) ;\ - OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ - curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ - chmod +x $(OPM) ;\ - } else -OPM = $(shell which opm) -endif -endif -# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). -# These images MUST exist in a registry and be pull-able. -BUNDLE_IMGS ?= $(BUNDLE_IMG) +# If we are not in build container, we need a workaround to get environment properly set +# Write to file, then include +$(shell mkdir -p out) +$(shell $(shell pwd)/common/scripts/setup_env.sh envfile > out/.env) +include out/.env +# An export free of arguments in a Makefile places all variables in the Makefile into the +# environment. This behavior may be surprising to many that use shell often, which simply +# displays the existing environment +export -# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). -CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) +export GOBIN ?= $(GOPATH)/bin +include Makefile.core.mk -# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. -ifneq ($(origin CATALOG_BASE_IMG), undefined) -FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) endif - -# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. -# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: -# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator -.PHONY: catalog-build -catalog-build: opm ## Build a catalog image. - $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) - -# Push the catalog image. -.PHONY: catalog-push -catalog-push: ## Push a catalog image. - $(MAKE) docker-push IMAGE=$(CATALOG_IMG) - - -##@ Linting - -FINDFILES=find . \( -path ./.git -o -path ./.github -o -path ./tmp \) -prune -o -type f -XARGS = xargs -0 -r - -.PHONY: lint-scripts -lint-scripts: ## runs linters against all shell scripts - @${FINDFILES} -name '*.sh' -print0 | ${XARGS} shellcheck - -.PHONY: lint-go -lint-go: ## runs linters against Go code - golangci-lint --version - golangci-lint run -c .golangci.yml -v - -.PHONY: lint-yaml -lint-yaml: ## runs linters against YAML files - @${FINDFILES} \( -name '*.yml' -o -name '*.yaml' \) -not -exec grep -q -e "{{" {} \; -print0 | ${XARGS} yamllint -f parsable - -.PHONY: lint-helm -lint-helm: ## runs linters against helm charts - @echo "Helm version: `helm version`" - @${FINDFILES} -name 'Chart.yaml' -path './resources/charts/v3.?/*' \ - -print0 | ${XARGS} -L 1 dirname | xargs -r helm lint --strict - -.PHONY: lint-bundle -lint-bundle: operator-sdk ## runs linters against OLM metadata bundle - $(OPERATOR_SDK) bundle validate bundle --select-optional suite=operatorframework - -.PHONY: lint-watches -lint-watches: ## checks if the operator watches all resource kinds present in Helm charts - @hack/lint-watches.sh - -.PHONY: lint -lint: lint-scripts lint-go lint-yaml lint-helm lint-bundle lint-watches ## runs all linters - -.SILENT: kustomize $(KUSTOMIZE) $(LOCALBIN) deploy-yaml - - -################################################################################ -# run an integration test on OCP -################################################################################ -.PHONY: test.integration.ocp -test.integration.ocp: - ${SOURCE_DIR}/tests/integration/integ-suite-ocp.sh - -################################################################################ -# run an integration test on Kind -################################################################################ -.PHONY: test.integration.kind -test.integration.kind: - ${SOURCE_DIR}/tests/integration/integ-suite-kind.sh - diff --git a/Makefile.core.mk b/Makefile.core.mk new file mode 100644 index 000000000..90ea1ff7d --- /dev/null +++ b/Makefile.core.mk @@ -0,0 +1,432 @@ +## Copyright 2019 Red Hat, Inc. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. + +-include Makefile.overrides + +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of your project. +# To re-generate a bundle for another specific version without changing the standard setup, you can: +# - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) +# - use environment variables to overwrite this value (e.g export VERSION=0.0.2) +VERSION ?= 3.0.0 +MINOR_VERSION := $(shell v='$(VERSION)'; echo "$${v%.*}") +MINIMUM_SUPPORTED_VERSION ?= v3.0 + +OPERATOR_NAME ?= sailoperator + +# Istio repository to pull charts from +ISTIO_REPOSITORY ?= maistra/istio +# the branch to use when updating the commit hash below +ISTIO_BRANCH_30 ?= maistra-3.0 +# the current commit of ${ISTIO_REPOSITORY} for v3.0. This will be used to copy charts +ISTIO_COMMIT_30 ?= ffdc450902e8fb7c2cd02eeca15d38d65b03f14b + +# Istio images names +ISTIO_CNI_IMAGE_NAME ?= install-cni +ISTIO_PILOT_IMAGE_NAME ?= pilot +ISTIO_PROXY_IMAGE_NAME ?= proxyv2 + +# GitHub creds +GITHUB_USER ?= maistra-bot +GITHUB_TOKEN ?= + +SOURCE_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) + +# Git repository state +ifndef GIT_TAG +GIT_TAG := $(shell git describe 2> /dev/null || echo "unknown") +endif +ifndef GIT_REVISION +GIT_REVISION := $(shell git rev-parse --verify HEAD 2> /dev/null || echo "unknown") +endif +ifndef GIT_STATUS +GIT_STATUS := $(shell git diff-index --quiet HEAD -- 2> /dev/null; if [ "$$?" = "0" ]; then echo Clean; else echo Modified; fi) +endif + +# Linker flags for the go builds +GO_MODULE = maistra.io/istio-operator +LD_EXTRAFLAGS = -X ${GO_MODULE}/pkg/version.buildVersion=${VERSION} +LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildGitRevision=${GIT_REVISION} +LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildTag=${GIT_TAG} +LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.buildStatus=${GIT_STATUS} +LD_EXTRAFLAGS += -X ${GO_MODULE}/pkg/version.minimumSupportedVersion=${MINIMUM_SUPPORTED_VERSION} +LD_FLAGS = -extldflags -static ${LD_EXTRAFLAGS} -s -w + +# Image hub to use +HUB ?= quay.io/maistra-dev +# Image tag to use +TAG ?= ${MINOR_VERSION}-latest +# Image base to use +IMAGE_BASE ?= istio-ubi9-operator +# Image URL to use all building/pushing image targets +IMAGE ?= ${HUB}/${IMAGE_BASE}:${TAG} +# Namespace to deploy the controller in +NAMESPACE ?= istio-operator +# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. +ENVTEST_K8S_VERSION = 1.26.0 + +# CHANNELS define the bundle channels used in the bundle. +# Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") +# To re-generate a bundle for other specific channels without changing the standard setup, you can: +# - use the CHANNELS as arg of the bundle target (e.g make bundle CHANNELS=candidate,fast,stable) +# - use environment variables to overwrite this value (e.g export CHANNELS="candidate,fast,stable") +CHANNELS ?= "3.0" +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=\"$(CHANNELS)\" +endif + +# DEFAULT_CHANNEL defines the default channel used in the bundle. +# Add a new line here if you would like to change its default config. (E.g DEFAULT_CHANNEL = "stable") +# To re-generate a bundle for any other default channel without changing the default setup, you can: +# - use the DEFAULT_CHANNEL as arg of the bundle target (e.g make bundle DEFAULT_CHANNEL=stable) +# - use environment variables to overwrite this value (e.g export DEFAULT_CHANNEL="stable") +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +# IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images. +# This variable is used to construct full image tags for bundle and catalog images. +# +# For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both +# quay.io/maistra-dev/istio-ubi9-operator-bundle:$VERSION and quay.io/maistra-dev/istio-ubi9-operator-catalog:$VERSION. +IMAGE_TAG_BASE ?= ${HUB}/${IMAGE_BASE} + +BUNDLE_MANIFEST_DATE := $(shell cat bundle/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml 2>/dev/null | grep createdAt | awk '{print $$2}') + +# BUNDLE_IMG defines the image:tag used for the bundle. +# You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=/:) +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +# BUNDLE_GEN_FLAGS are the flags passed to the operator-sdk generate bundle command +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +# USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +# You can enable this value if you would like to use SHA Based Digests +# To enable set flag to true +USE_IMAGE_DIGESTS ?= false +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +TODAY ?= $(shell date -I) + +# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) +ifeq (,$(shell go env GOBIN)) +GOBIN=$(shell go env GOPATH)/bin +else +GOBIN=$(shell go env GOBIN) +endif + +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -o pipefail +.SHELLFLAGS = -ec + +.PHONY: all +all: build + +##@ Testing + +.PHONY: test +test: envtest ## Run tests. + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... -coverprofile cover.out + +.PHONY: test.scorecard ## Runs the operator scorecard test. Needs a valid k8s cluster as pointed by the KUBECONFIG variable +test.scorecard: operator-sdk + $(OPERATOR_SDK) scorecard bundle + +.PHONY: test.integration.ocp +test.integration.ocp: + ${SOURCE_DIR}/tests/integration/integ-suite-ocp.sh + +.PHONY: test.integration.kind +test.integration.kind: + ${SOURCE_DIR}/tests/integration/integ-suite-kind.sh + +##@ Build + +.PHONY: build +build: ## Build manager binary. + CGO_ENABLED=0 go build -o bin/manager -ldflags '${LD_FLAGS}' main.go + +.PHONY: run +run: gen ## Run a controller from your host. + POD_NAMESPACE=${NAMESPACE} go run ./main.go --config-file=./hack/config.properties --resource-directory=./resources + +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + docker build -t ${IMAGE} --build-arg GIT_TAG=${GIT_TAG} --build-arg GIT_REVISION=${GIT_REVISION} --build-arg GIT_STATUS=${GIT_STATUS} . + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMAGE} + +.PHONY: docker-push-nightly ## Build and push nightly docker image with the manager. +docker-push-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) +docker-push-nightly: docker-build + docker push ${IMAGE} + docker tag ${IMAGE} $(HUB)/istio-ubi9-operator:$(MINOR_VERSION)-latest + docker push $(HUB)/istio-ubi9-operator:$(MINOR_VERSION)-latest + +# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple +# architectures. (i.e. make docker-buildx IMAGE=myregistry/mypoperator:0.0.1). To use this option you need to: +# - able to use docker buildx . More info: https://docs.docker.com/build/buildx/ +# - have enable BuildKit, More info: https://docs.docker.com/develop/develop-images/build_enhancements/ +# - be able to push the image for your registry (i.e. if you do not inform a valid value via IMAGE=> then the export will fail) +# To properly provided solutions that supports more than one platform you should use this option. +PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le +.PHONY: docker-buildx +docker-buildx: test ## Build and push docker image for the manager for cross-platform support + # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile + sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross + - docker buildx create --name project-v3-builder + docker buildx use project-v3-builder + - docker buildx build --push --platform=$(PLATFORMS) --tag ${IMAGE} -f Dockerfile.cross . + - docker buildx rm project-v3-builder + rm Dockerfile.cross + +##@ Deployment + +ifndef ignore-not-found + ignore-not-found = false +endif + +.PHONY: install +install: gen-manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + kubectl create ns ${NAMESPACE} || echo "namespace ${NAMESPACE} already exists" + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUSTOMIZE) build config/crd | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: deploy +deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + $(info NAMESPACE: $(NAMESPACE)) + $(MAKE) -s deploy-yaml | kubectl apply -f - + +.PHONY: deploy-yaml +deploy-yaml: kustomize ## Outputs YAML manifests needed to deploy the controller + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMAGE} + cd config/default && $(KUSTOMIZE) edit set namespace ${NAMESPACE} + $(KUSTOMIZE) build config/default + +.PHONY: deploy-olm +deploy-olm: bundle bundle-build bundle-push ## Builds and pushes the operator OLM bundle and then deploys the operator using OLM + kubectl create ns ${NAMESPACE} || echo "namespace ${NAMESPACE} already exists" + $(OPERATOR_SDK) run bundle $(BUNDLE_IMG) -n ${NAMESPACE} + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(MAKE) deploy-yaml | kubectl delete --ignore-not-found=$(ignore-not-found) -f - + +.PHONY: undeploy-olm +undeploy-olm: operator-sdk ## Undeploys the operator from the cluster (used only if operator was installed via OLM) + $(OPERATOR_SDK) cleanup sailoperator --delete-all -n ${NAMESPACE} + +.PHONY: deploy-example +deploy-example: deploy-example-openshift + +.PHONY: deploy-example-openshift +deploy-example-openshift: ## Deploy an example Istio resource on OpenShift + kubectl create ns istio-system || echo "namespace istio-system already exists" + kubectl apply -n istio-system -f config/samples/istio-sample-openshift.yaml + +.PHONY: deploy-example-kubernetes +deploy-example-kubernetes: ## Deploy an example Istio resource on Kubernetes + kubectl create ns istio-system || echo "namespace istio-system already exists" + kubectl apply -n istio-system -f config/samples/istio-sample-kubernetes.yaml + +##@ Generated Code & Resources + +.PHONY: gen-manifests +gen-manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases + +.PHONY: gen-code +gen-code: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." + +.PHONY: gen-charts +gen-charts: ## Pull charts from maistra/istio repository + hack/download-charts.sh v${MINOR_VERSION} https://github.com/${ISTIO_REPOSITORY} ${ISTIO_COMMIT_30} + +.PHONY: gen ## Generate everything +gen: controller-gen gen-manifests gen-code gen-charts bundle + +.PHONY: gen-check +gen-check: gen restore-manifest-dates check-clean-repo ## Verifies that changes in generated resources have been checked in + +.PHONY: restore-manifest-dates +restore-manifest-dates: +ifneq "${BUNDLE_MANIFEST_DATE}" "" + @sed -i -e "s/\(createdAt:\).*/\1 \"${BUNDLE_MANIFEST_DATE}\"/" bundle/manifests/${OPERATOR_NAME}.clusterserviceversion.yaml +endif + +.PHONY: update-istio +update-istio: ## Updates the Istio commit hash to latest on ${ISTIO_BRANCH_30} + $(eval ISTIO_COMMIT_30=$(shell git ls-remote https://github.com/${ISTIO_REPOSITORY}.git | grep ${ISTIO_BRANCH_30} | cut -f 1)) + @echo Updating to ${ISTIO_REPOSITORY}@${ISTIO_COMMIT_30}; sed -i -e "s/^\(ISTIO_COMMIT_30 ?= \).*$$/\1${ISTIO_COMMIT_30}/g" Makefile + +.PHONY: patch-istio-images +patch-istio-images: ## Patch the Istio images in the ClusterServiceVersion with the right tags + sed -i -e "s|images3_0.cni: .*|images3_0.cni: $(HUB)/$(ISTIO_CNI_IMAGE_NAME):$(TAG)|" \ + -e "s|images3_0.istiod: .*|images3_0.istiod: $(HUB)/$(ISTIO_PILOT_IMAGE_NAME):$(TAG)|" \ + -e "s|images3_0.proxy: .*|images3_0.proxy: $(HUB)/$(ISTIO_PROXY_IMAGE_NAME):$(TAG)|" \ + $(shell ls bundle/manifests/*.clusterserviceversion.yaml) + +##@ Build Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +OPERATOR_SDK_VERSION ?= v1.31.0 +KUSTOMIZE_VERSION ?= v3.8.7 +CONTROLLER_TOOLS_VERSION ?= v0.11.1 + +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. If wrong version is installed, it will be removed before downloading. +$(KUSTOMIZE): $(LOCALBIN) + if test -x $(LOCALBIN)/kustomize && ! $(LOCALBIN)/kustomize version | grep -q $(KUSTOMIZE_VERSION) > /dev/stderr; then \ + echo "$(LOCALBIN)/kustomize version is not expected $(KUSTOMIZE_VERSION). Removing it before installing." > /dev/stderr; \ + rm -rf $(LOCALBIN)/kustomize; \ + fi + test -s $(LOCALBIN)/kustomize || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) > /dev/stderr; } + +.PHONY: operator-sdk +operator-sdk: $(OPERATOR_SDK) +operator-sdk: OS=$(shell go env GOOS) +operator-sdk: ARCH=$(shell go env GOARCH) +$(OPERATOR_SDK): $(LOCALBIN) + @if test -x $(LOCALBIN)/operator-sdk && ! $(LOCALBIN)/operator-sdk version | grep -q $(OPERATOR_SDK_VERSION); then \ + echo "$(LOCALBIN)/operator-sdk version is not expected $(OPERATOR_SDK_VERSION). Removing it before installing."; \ + rm -rf $(LOCALBIN)/operator-sdk; \ + fi + test -s $(LOCALBIN)/operator-sdk || \ + curl -sSLo $(LOCALBIN)/operator-sdk https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(OS)_$(ARCH) && \ + chmod +x $(LOCALBIN)/operator-sdk; + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. If wrong version is installed, it will be overwritten. +$(CONTROLLER_GEN): $(LOCALBIN) + test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \ + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + +.PHONY: bundle +bundle: gen kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + $(OPERATOR_SDK) generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMAGE) + sed -i "s|^\( containerImage:\).*$$|\1 ${IMAGE}|g" config/manifests/bases/${OPERATOR_NAME}.clusterserviceversion.yaml + $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) + $(OPERATOR_SDK) bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMAGE=$(BUNDLE_IMG) + +.PHONY: bundle-publish +bundle-publish: patch-istio-images ## Create a PR for publishing in OperatorHub + export GIT_USER=$(GITHUB_USER); \ + export GITHUB_TOKEN=$(GITHUB_TOKEN); \ + export OPERATOR_VERSION=$(OPERATOR_VERSION); \ + export OPERATOR_NAME=$(OPERATOR_NAME); \ + ./hack/operatorhub/publish-bundle.sh + +.PHONY: bundle-nightly ## Generate nightly bundle +bundle-nightly: + $(MAKE) bundle VERSION=${VERSION}-nightly-${TODAY} CHANNELS=$(MINOR_VERSION)-nightly TAG=$(MINOR_VERSION)-nightly-$(TODAY) + +.PHONY: bundle-publish-nightly +bundle-publish-nightly: OPERATOR_VERSION=$(VERSION)-nightly-$(TODAY) +bundle-publish-nightly: TAG=$(MINOR_VERSION)-nightly-$(TODAY) +bundle-publish-nightly: bundle-nightly bundle-publish + +.PHONY: opm +OPM = ./bin/opm +opm: ## Download opm locally if necessary. +ifeq (,$(wildcard $(OPM))) +ifeq (,$(shell which opm 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPM)) ;\ + OS=$(shell go env GOOS) && ARCH=$(shell go env GOARCH) && \ + curl -sSLo $(OPM) https://github.com/operator-framework/operator-registry/releases/download/v1.23.0/$${OS}-$${ARCH}-opm ;\ + chmod +x $(OPM) ;\ + } +else +OPM = $(shell which opm) +endif +endif + +# A comma-separated list of bundle images (e.g. make catalog-build BUNDLE_IMGS=example.com/operator-bundle:v0.1.0,example.com/operator-bundle:v0.2.0). +# These images MUST exist in a registry and be pull-able. +BUNDLE_IMGS ?= $(BUNDLE_IMG) + +# The image tag given to the resulting catalog image (e.g. make catalog-build CATALOG_IMG=example.com/operator-catalog:v0.2.0). +CATALOG_IMG ?= $(IMAGE_TAG_BASE)-catalog:v$(VERSION) + +# Set CATALOG_BASE_IMG to an existing catalog image tag to add $BUNDLE_IMGS to that image. +ifneq ($(origin CATALOG_BASE_IMG), undefined) +FROM_INDEX_OPT := --from-index $(CATALOG_BASE_IMG) +endif + +# Build a catalog image by adding bundle images to an empty catalog using the operator package manager tool, 'opm'. +# This recipe invokes 'opm' in 'semver' bundle add mode. For more information on add modes, see: +# https://github.com/operator-framework/community-operators/blob/7f1438c/docs/packaging-operator.md#updating-your-existing-operator +.PHONY: catalog-build +catalog-build: opm ## Build a catalog image. + $(OPM) index add --container-tool docker --mode semver --tag $(CATALOG_IMG) --bundles $(BUNDLE_IMGS) $(FROM_INDEX_OPT) + +# Push the catalog image. +.PHONY: catalog-push +catalog-push: ## Push a catalog image. + $(MAKE) docker-push IMAGE=$(CATALOG_IMG) + + +##@ Linting + +.PHONY: lint-bundle +lint-bundle: operator-sdk ## runs linters against OLM metadata bundle + $(OPERATOR_SDK) bundle validate bundle --select-optional suite=operatorframework + +.PHONY: lint-watches +lint-watches: ## checks if the operator watches all resource kinds present in Helm charts + @hack/lint-watches.sh + +.PHONY: lint +lint: lint-scripts lint-go lint-yaml lint-helm lint-bundle lint-watches ## runs all linters + +.SILENT: kustomize $(KUSTOMIZE) $(LOCALBIN) deploy-yaml + +include common/Makefile.common.mk diff --git a/Makefile.overrides.mk b/Makefile.overrides.mk new file mode 100644 index 000000000..bafbef6bd --- /dev/null +++ b/Makefile.overrides.mk @@ -0,0 +1,13 @@ +.DEFAULT_GOAL := default +BUILD_WITH_CONTAINER ?= 1 +CONTAINER_OPTIONS = --mount type=bind,source=/tmp,destination=/tmp --net=host + +export COMMONFILES_POSTPROCESS = tools/commonfiles-postprocess.sh + +ifeq ($(BUILD_WITH_CONTAINER),1) +# create phony targets for the top-level items in the repo +PHONYS := $(shell ls | grep -v Makefile) +.PHONY: $(PHONYS) +$(PHONYS): + @$(MAKE_DOCKER) $@ +endif diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 000000000..505913929 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,7 @@ +# Support + +Here are some resources to help you understand and use Istio: + +- For in-depth information about how to use Istio, visit [istio.io](https://istio.io) +- To ask questions and get assistance from our community, visit [discuss.istio.io](https://discuss.istio.io) +- To learn how to participate in our overall community, visit [our community page](https://istio.io/about/community) diff --git a/common/.commonfiles.sha b/common/.commonfiles.sha new file mode 100644 index 000000000..8f72d3787 --- /dev/null +++ b/common/.commonfiles.sha @@ -0,0 +1 @@ +37b1df7c218df019feaa525e91c35265ba9a06f7 diff --git a/common/Makefile.common.mk b/common/Makefile.common.mk new file mode 100644 index 000000000..648e197bc --- /dev/null +++ b/common/Makefile.common.mk @@ -0,0 +1,121 @@ +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./out -o -path ./.github -o -path ./licenses -o -path ./vendor \) -prune -o -type f +XARGS = xargs -0 -r + +lint-dockerfiles: + @${FINDFILES} -name 'Dockerfile*' -print0 | ${XARGS} hadolint -c ./common/config/.hadolint.yml + +lint-scripts: + @${FINDFILES} -name '*.sh' -print0 | ${XARGS} -n 256 shellcheck + +lint-yaml: + @${FINDFILES} \( -name '*.yml' -o -name '*.yaml' \) -not -exec grep -q -e "{{" {} \; -print0 | ${XARGS} yamllint -c ./common/config/.yamllint.yml + +lint-helm: + @${FINDFILES} -name 'Chart.yaml' -print0 | ${XARGS} -L 1 dirname | xargs -r helm lint --strict + +lint-copyright-banner: + @${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' -o -name '*.rs' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\ + ${XARGS} common/scripts/lint_copyright_banner.sh + +fix-copyright-banner: + @${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' -o -name '*.rs' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\ + ${XARGS} common/scripts/fix_copyright_banner.sh + +lint-go: + @${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/lint_go.sh + +lint-python: + @${FINDFILES} -name '*.py' \( ! \( -name '*_pb2.py' \) \) -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d + +lint-markdown: + @${FINDFILES} -name '*.md' -print0 | ${XARGS} mdl --ignore-front-matter --style common/config/mdl.rb + +lint-links: + @${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect --white-list ${MARKDOWN_LINT_ALLOWLIST} + +lint-sass: + @${FINDFILES} -name '*.scss' -print0 | ${XARGS} sass-lint -c common/config/sass-lint.yml --verbose + +lint-typescript: + @${FINDFILES} -name '*.ts' -print0 | ${XARGS} tslint -c common/config/tslint.json + +lint-licenses: + @if test -d licenses; then license-lint --config common/config/license-lint.yml; fi + +lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript lint-licenses + +tidy-go: + @find -name go.mod -execdir go mod tidy \; + +mod-download-go: + @-GOFLAGS="-mod=readonly" find -name go.mod -execdir go mod download \; +# go mod tidy is needed with Golang 1.16+ as go mod download affects go.sum +# https://github.com/golang/go/issues/43994 + @find -name go.mod -execdir go mod tidy \; + +format-go: tidy-go + @${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/format_go.sh + +format-python: + @${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --aggressive --aggressive -i + +dump-licenses: mod-download-go + @license-lint --config common/config/license-lint.yml --report + +dump-licenses-csv: mod-download-go + @license-lint --config common/config/license-lint.yml --csv + +mirror-licenses: mod-download-go + @rm -fr licenses + @license-lint --mirror + +TMP := $(shell mktemp -d -u) +UPDATE_BRANCH ?= "master" + +BUILD_TOOLS_ORG ?= "istio" + +update-common: + @mkdir -p $(TMP) + @git clone -q --depth 1 --single-branch --branch $(UPDATE_BRANCH) https://github.com/$(BUILD_TOOLS_ORG)/common-files $(TMP)/common-files + @cd $(TMP)/common-files ; git rev-parse HEAD >files/common/.commonfiles.sha + @rm -fr common + @CONTRIB_OVERRIDE=$(shell grep -l "istio/community/blob/master/CONTRIBUTING.md" CONTRIBUTING.md) + @if [ "$(CONTRIB_OVERRIDE)" != "CONTRIBUTING.md" ]; then\ + rm $(TMP)/common-files/files/CONTRIBUTING.md;\ + fi + @cp -a $(TMP)/common-files/files/* $(TMP)/common-files/files/.devcontainer $(TMP)/common-files/files/.gitattributes $(shell pwd) + @rm -fr $(TMP)/common-files + @$(or $(COMMONFILES_POSTPROCESS), true) + +check-clean-repo: + @common/scripts/check_clean_repo.sh + +tidy-docker: + @docker image prune --all --force --filter="label=io.istio.repo=https://github.com/istio/tools" --filter="label!=io.istio.version=$(IMAGE_VERSION)" + +# help works by looking over all Makefile includes matching `target: ## comment` regex and outputting them +help: ## Show this help + @egrep -h '^[a-zA-Z_\.-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' + +.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-python lint-helm lint-markdown lint-sass lint-typescript lint-all format-go format-python update-common lint-licenses dump-licenses dump-licenses-csv check-clean-repo tidy-docker help tidy-go mod-download-go diff --git a/common/config/.golangci-format.yml b/common/config/.golangci-format.yml new file mode 100644 index 000000000..cee514d6a --- /dev/null +++ b/common/config/.golangci-format.yml @@ -0,0 +1,59 @@ +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +service: + # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo. + golangci-lint-version: 1.54.x # use the fixed version to not introduce new linters unexpectedly +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 20m + build-tags: + - integ + - integfuzz + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + - genfiles$ + - vendor$ + + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: + - ".*\\.pb\\.go" + - ".*\\.gen\\.go" + +linters: + disable-all: true + enable: + - goimports + - gofumpt + - gci + fast: false + +linters-settings: + gci: + sections: + - standard # Captures all standard packages if they do not match another section. + - default # Contains all imports that could not be matched to another section type. + - prefix(istio.io/) # Groups all imports with the specified Prefix. + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: istio.io/ + +issues: + + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-per-linter: 0 + + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 diff --git a/common/config/.golangci.yml b/common/config/.golangci.yml new file mode 100644 index 000000000..fe77a42ec --- /dev/null +++ b/common/config/.golangci.yml @@ -0,0 +1,331 @@ +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". +service: + # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo. + golangci-lint-version: 1.54.x # use the fixed version to not introduce new linters unexpectedly +run: + # timeout for analysis, e.g. 30s, 5m, default is 1m + deadline: 20m + build-tags: + - integ + - integfuzz + # which dirs to skip: they won't be analyzed; + # can use regexp here: generated.*, regexp is applied on full path; + # default value is empty list, but next dirs are always skipped independently + # from this option's value: + # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ + skip-dirs: + - genfiles$ + - vendor$ + # which files to skip: they will be analyzed, but issues from them + # won't be reported. Default value is empty list, but there is + # no need to include all autogenerated files, we confidently recognize + # autogenerated files. If it's not please let us know. + skip-files: + - ".*\\.pb\\.go" + - ".*\\.gen\\.go" +linters: + disable-all: true + enable: + - errcheck + - exportloopref + - depguard + - gocritic + - gofumpt + - goimports + - revive + - gosimple + - govet + - ineffassign + - lll + - misspell + - staticcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - gci + - gosec + fast: false +linters-settings: + errcheck: + # report about not checking of errors in type assertions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: false + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: false + govet: + # report about shadowed variables + check-shadowing: false + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: istio.io/ + maligned: + # print struct with more effective memory layout or not, false by default + suggest-new: true + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + ignore-words: + - cancelled + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 160 + # tab width in spaces. Default to 1. + tab-width: 1 + revive: + ignore-generated-header: false + severity: "warning" + confidence: 0.0 + error-code: 2 + warning-code: 1 + rules: + - name: blank-imports + - name: context-keys-type + - name: time-naming + - name: var-declaration + - name: unexported-return + - name: errorf + - name: context-as-argument + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: increment-decrement + - name: var-naming + - name: package-comments + - name: range + - name: receiver-naming + - name: indent-error-flow + - name: superfluous-else + - name: modifies-parameter + - name: unreachable-code + - name: struct-tag + - name: constant-logical-expr + - name: bool-literal-in-expr + - name: redefines-builtin-id + - name: imports-blacklist + - name: range-val-in-closure + - name: range-val-address + - name: waitgroup-by-value + - name: atomic + - name: call-to-gc + - name: duplicated-imports + - name: string-of-int + - name: defer + arguments: + - - "call-chain" + - name: unconditional-recursion + - name: identical-branches + # the following rules can be enabled in the future + # - name: empty-lines + # - name: confusing-results + # - name: empty-block + # - name: get-return + # - name: confusing-naming + # - name: unexported-naming + # - name: early-return + # - name: unused-parameter + # - name: unnecessary-stmt + # - name: deep-exit + # - name: import-shadowing + # - name: modifies-value-receiver + # - name: unused-receiver + # - name: bare-return + # - name: flag-parameter + # - name: unhandled-error + # - name: if-return + unused: + # treat code as a program (not a library) and report unused exported identifiers; default is false. + # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + unparam: + # call graph construction algorithm (cha, rta). In general, use cha for libraries, + # and rta for programs with main packages. Default is cha. + algo: cha + # Inspect exported functions, default is false. Set to true if no external program/library imports your code. + # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: + # if it's called for subdir of a project it can't find external interfaces. All text editor integrations + # with golangci-lint call it on a directory with the changed file. + check-exported: false + gci: + sections: + - standard # Captures all standard packages if they do not match another section. + - default # Contains all imports that could not be matched to another section type. + - prefix(istio.io/) # Groups all imports with the specified Prefix. + gocritic: + enabled-checks: + - appendCombine + - argOrder + - assignOp + - badCond + - boolExprSimplify + - builtinShadow + - captLocal + - caseOrder + - codegenComment + - commentedOutCode + - commentedOutImport + - defaultCaseOrder + - deprecatedComment + - docStub + - dupArg + - dupBranchBody + - dupCase + - dupSubExpr + - elseif + - emptyFallthrough + - equalFold + - flagDeref + - flagName + - hexLiteral + - indexAlloc + - initClause + - methodExprCall + - nilValReturn + - octalLiteral + - offBy1 + - rangeExprCopy + - regexpMust + - sloppyLen + - stringXbytes + - switchTrue + - typeAssertChain + - typeSwitchVar + - typeUnparen + - underef + - unlambda + - unnecessaryBlock + - unslice + - valSwap + - weakCond + # Unused + # - yodaStyleExpr + # - appendAssign + # - commentFormatting + # - emptyStringTest + # - exitAfterDefer + # - ifElseChain + # - hugeParam + # - importShadow + # - nestingReduce + # - paramTypeCombine + # - ptrToRefParam + # - rangeValCopy + # - singleCaseSwitch + # - sloppyReassign + # - unlabelStmt + # - unnamedResult + # - wrapperFunc + depguard: + rules: + DenyGogoProtobuf: + files: + - $all + deny: + - pkg: github.com/gogo/protobuf + desc: "gogo/protobuf is deprecated, use golang/protobuf" + # deny for all go files + AllGoFiles: + files: + - $all + deny: + - pkg: golang.org/x/net/http2/h2c + desc: "h2c.NewHandler is unsafe; use wrapper istio.io/istio/pkg/h2c" + - pkg: github.com/golang/protobuf/jsonpb + desc: "don't use the jsonpb package directly; use util/protomarshal instead" + - pkg: google.golang.org/protobuf/encoding/protojson + desc: "don't use the protojson package directly; use util/protomarshal instead" + - pkg: gomodules.xyz/jsonpatch/v3 + desc: "don't use v3; v2 is orders of magnitude higher performance" + - pkg: k8s.io/apimachinery/pkg/util/sets + desc: "use istio.io/istio/pkg/sets" + - pkg: k8s.io/utils/env + desc: "use istio.io/istio/pkg/env" + - pkg: k8s.io/utils/strings/slices + desc: "use istio.io/istio/pkg/slices" + - pkg: k8s.io/utils/pointer + desc: "use istio.io/istio/pkg/ptr" + - pkg: go.opencensus.io + desc: "do not use OpenCensus; use OpenTelemetry instead" + DenyOperatorAndIstioctl: + files: + # Tests can do anything + - "!$test" + # Main code should only be used by appropriate binaries + - "!**/operator/**" + - "!**/istioctl/**" + - "!**/tools/bug-report/**" + # This should only really import operator API, but that is hard to express without a larger refactoring + - "!**/pkg/kube/**" + - "!**/pkg/url/**" + - "!**/pkg/test/framework/**" + - "!**/tests/fuzz/**" + deny: + - pkg: istio.io/istio/operator + desc: "operator should not be imported" + - pkg: istio.io/istio/istioctl + desc: "istioctl should not be imported" + DenyOpenTelemetry: + files: + - $all + - "!**/pkg/monitoring/**" + - "!**/pkg/tracing/**" + deny: + - pkg: go.opentelemetry.io/otel + desc: "do not use OpenTelemetry directly; use pkg/monitoring" + - pkg: go.opentelemetry.io/otel/metric + desc: "do not use OpenTelemetry directly; use pkg/monitoring" + gosec: + includes: + - G401 + - G402 + - G404 +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + - composite literal uses unkeyed fields + exclude-rules: + # Exclude some linters from running on test files. + - path: _test\.go$|^tests/|^samples/ + linters: + - errcheck + - maligned + # We need to use the deprecated module since the jsonpb replacement is not backwards compatible. + - linters: + - staticcheck + text: "SA1019: package github.com/golang/protobuf/jsonpb" + - linters: + - staticcheck + text: 'SA1019: "github.com/golang/protobuf/jsonpb"' + - linters: + - revive + text: "struct-tag: unknown option 'inline' in JSON tag" + # Don't check *_types.go files for lines that are too long (the linter complains about +kubebuilder:printcolumn lines) + - path: _types.go$ + linters: + - lll + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: true + # Maximum issues count per one linter. Set to 0 to disable. Default is 50. + max-per-linter: 0 + # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. + max-same-issues: 0 diff --git a/common/config/.hadolint.yml b/common/config/.hadolint.yml new file mode 100644 index 000000000..3e4e1cbab --- /dev/null +++ b/common/config/.hadolint.yml @@ -0,0 +1,15 @@ +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +ignored: + - DL3008 + - DL3059 + +trustedRegistries: + - gcr.io + - docker.io + - quay.io diff --git a/common/config/.yamllint.yml b/common/config/.yamllint.yml new file mode 100644 index 000000000..c2f21b581 --- /dev/null +++ b/common/config/.yamllint.yml @@ -0,0 +1,29 @@ +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +rules: + braces: disable + brackets: disable + colons: enable + commas: disable + comments: disable + comments-indentation: disable + document-end: disable + document-start: disable + empty-lines: disable + empty-values: disable + hyphens: enable + indentation: disable + key-duplicates: enable + key-ordering: disable + line-length: disable + new-line-at-end-of-file: disable + new-lines: enable + octal-values: disable + quoted-strings: disable + trailing-spaces: disable + truthy: disable diff --git a/common/config/license-lint.yml b/common/config/license-lint.yml new file mode 100644 index 000000000..ef4859462 --- /dev/null +++ b/common/config/license-lint.yml @@ -0,0 +1,128 @@ +unrestricted_licenses: + - Apache-2.0 + - CC-BY-3.0 + - ISC + - AFL-2.1 + - AFL-3.0 + - Artistic-1.0 + - Artistic-2.0 + - Apache-1.1 + - BSD-1-Clause + - BSD-2-Clause + - BSD-3-Clause + - 0BSD + - FTL + - LPL-1.02 + - MS-PL + - MIT + - NCSA + - OpenSSL + - PHP-3.0 + - TCP-wrappers + - W3C + - Xnet + - Zlib + +reciprocal_licenses: + - CC0-1.0 + - APSL-2.0 + - CDDL-1.0 + - CDDL-1.1 + - CPL-1.0 + - EPL-1.0 + - IPL-1.0 + - MPL-1.0 + - MPL-1.1 + - MPL-2.0 + - MPL-2.0-no-copyleft-exception + - Ruby + +restricted_licenses: + - GPL-1.0-only + - GPL-1.0-or-later + - GPL-2.0-only + - GPL-2.0-or-later + - GPL-3.0-only + - GPL-3.0-or-later + - LGPL-2.0-only + - LGPL-2.0-or-later + - LGPL-2.1-only + - LGPL-2.1-or-later + - LGPL-3.0-only + - LGPL-3.0-or-later + - NPL-1.0 + - NPL-1.1 + - OSL-1.0 + - OSL-1.1 + - OSL-2.0 + - OSL-2.1 + - OSL-3.0 + - QPL-1.0 + - Sleepycat + +allowlisted_modules: +# MIT: https://github.com/ghodss/yaml/blob/master/LICENSE +- github.com/ghodss/yaml + +# BSD: https://github.com/gogo/protobuf/blob/master/LICENSE +- github.com/gogo/protobuf + +# BSD: https://github.com/magiconair/properties/blob/master/LICENSE.md +- github.com/magiconair/properties + +# Apache 2.0 +- github.com/spf13/cobra +- github.com/spf13/afero + +# Public domain: https://github.com/xi2/xz/blob/master/LICENSE +- github.com/xi2/xz + +# Helm is Apache 2.0: https://github.com/helm/helm/blob/master/LICENSE +# However, it has a bunch of LICENSE test files that our linter fails to understand +- helm.sh/helm/v3 + +# https://github.com/pelletier/go-toml/blob/master/LICENSE +# Uses MIT for everything, except a few files copied from +# google's civil library that uses Apache 2 +- github.com/pelletier/go-toml + +# https://github.com/xeipuuv/gojsonpointer/blob/master/LICENSE-APACHE-2.0.txt +- github.com/xeipuuv/gojsonpointer +# https://github.com/xeipuuv/gojsonreference/blob/master/LICENSE-APACHE-2.0.txt +- github.com/xeipuuv/gojsonreference +# Apache 2.0: https://github.com/xeipuuv/gojsonschema/blob/master/LICENSE-APACHE-2.0.txt +- github.com/xeipuuv/gojsonschema + +# Apache 2.0 (but missing appendix): https://github.com/garyburd/redigo/blob/master/LICENSE +- github.com/garyburd/redigo +- github.com/gomodule/redigo + +# Apache 2.0 +# github.com/ghodss/yaml: MIT / BSD-3 +# github.com/gogo/protobuf: BSD-3 +# github.com/jmespath/go-jmespath: Apache 2.0 +# sigs.k8s.io/yaml: MIT / BSD-3 +- github.com/tektoncd/pipeline + +# MIT: https://github.com/kubernetes-sigs/yaml/blob/master/LICENSE +- sigs.k8s.io/yaml + +# https://github.com/go-errors/errors/blob/master/LICENSE.MIT +- github.com/go-errors/errors + +# runc is Apache 2.0: https://github.com/opencontainers/runc/blob/master/LICENSE +# but it contains BSD dep which our linter fails to understand: https://github.com/opencontainers/runc/blob/v0.1.1/Godeps/_workspace/src/github.com/golang/protobuf/LICENSE +- github.com/opencontainers/runc + +# MIT: https://github.com/felixge/fgprof/blob/master/LICENSE.txt +- github.com/felixge/fgprof + +# Apache 2.0 +- github.com/google/pprof + +# MIT: https://github.com/invopop/yaml/blob/v0.1.0/LICENSE +- github.com/invopop/yaml + +# Simplified BSD (BSD-2-Clause): https://github.com/russross/blackfriday/blob/master/LICENSE.txt +- github.com/russross/blackfriday +- github.com/russross/blackfriday/v2 \ No newline at end of file diff --git a/common/config/mdl.rb b/common/config/mdl.rb new file mode 100644 index 000000000..8764f94d7 --- /dev/null +++ b/common/config/mdl.rb @@ -0,0 +1,12 @@ +all +rule 'MD002', :level => 1 +rule 'MD007', :indent => 4 +rule 'MD013', :line_length => 160, :code_blocks => false, :tables => false +rule 'MD026', :punctuation => ".,;:!" +exclude_rule 'MD013' +exclude_rule 'MD014' +exclude_rule 'MD030' +exclude_rule 'MD032' +exclude_rule 'MD033' +exclude_rule 'MD041' +exclude_rule 'MD046' diff --git a/common/config/sass-lint.yml b/common/config/sass-lint.yml new file mode 100644 index 000000000..da43ee79c --- /dev/null +++ b/common/config/sass-lint.yml @@ -0,0 +1,98 @@ +######################### +## Config for sass-lint +######################### +# Linter Options +options: + # Don't merge default rules + merge-default-rules: false + # Raise an error if more than 50 warnings are generated + max-warnings: 500 +# Rule Configuration +rules: + attribute-quotes: + - 2 + - + include: false + bem-depth: 2 + border-zero: 2 + brace-style: 2 + class-name-format: 2 + clean-import-paths: 2 + declarations-before-nesting: 2 + empty-args: 2 + empty-line-between-blocks: 2 + extends-before-declarations: 2 + extends-before-mixins: 2 + final-newline: 2 + force-attribute-nesting: 0 + force-element-nesting: 0 + force-pseudo-nesting: 0 + function-name-format: 2 + hex-length: 0 + hex-notation: 2 + id-name-format: 2 + indentation: + - 2 + - + size: 4 + leading-zero: + - 2 + - + include: false + max-file-line-count: 0 + max-file-length: 0 + mixins-before-declarations: 2 + no-attribute-selectors: 0 + no-color-hex: 0 + no-color-keywords: 0 + no-color-literals: 0 + no-combinators: 0 + no-css-comments: 2 + no-debug: 2 + no-disallowed-properties: 2 + no-duplicate-properties: 2 + no-empty-rulesets: 2 + no-extends: 2 + no-ids: 0 + no-invalid-hex: 2 + no-important: 0 + no-mergeable-selectors: 2 + no-misspelled-properties: 2 + no-qualifying-elements: 0 + no-trailing-whitespace: 2 + no-trailing-zero: 2 + no-transition-all: 0 + no-url-domains: 2 + no-url-protocols: 2 + no-warn: 2 + one-declaration-per-line: 2 + placeholder-in-extend: 2 + placeholder-name-format: 2 + property-sort-order: 0 + property-units: 2 + pseudo-element: 2 + quotes: + - 2 + - + style: double + shorthand-values: 2 + single-line-per-selector: 0 + space-after-bang: 2 + space-after-colon: 2 + space-after-comma: 2 + space-around-operator: 2 + space-before-bang: 2 + space-before-brace: 2 + space-before-colon: 2 + space-between-parens: 2 + trailing-semicolon: 2 + url-quotes: 2 + variable-for-property: + - 0 + - + properties: + - color + - background-color + - fill + variable-name-format: 0 + zero-unit: 2 diff --git a/common/config/tslint.json b/common/config/tslint.json new file mode 100644 index 000000000..6db4bb19d --- /dev/null +++ b/common/config/tslint.json @@ -0,0 +1,25 @@ +{ + "defaultSeverity": "error", + "extends": [ + "tslint:recommended" + ], + "rules": { + "max-line-length": { + "options": [160] + }, + "arrow-parens": false, + "new-parens": true, + "no-arg": true, + "no-bitwise": true, + "no-conditional-assignment": true, + "no-consecutive-blank-lines": true, + "no-console": { + "severity": "warning", + "options": ["debug", "info", "log", "time", "timeEnd", "trace"] + }, + "no-shadowed-variable": false, + "eofline": false + }, + "jsRules": {}, + "rulesDirectory": [] +} \ No newline at end of file diff --git a/common/scripts/check_clean_repo.sh b/common/scripts/check_clean_repo.sh new file mode 100755 index 000000000..abe067398 --- /dev/null +++ b/common/scripts/check_clean_repo.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# Copyright Istio Authors + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function write_patch_file() { + if [ -z "${ARTIFACTS}" ]; then + return 0 + fi + + PATCH_NAME="check-clean-repo-diff.patch" + PATCH_OUT="${ARTIFACTS}/${PATCH_NAME}" + git diff > "${PATCH_OUT}" + + [ -n "${JOB_NAME}" ] && [ -n "${BUILD_ID}" ] + IN_PROW="$?" + + # Don't persist large diffs (30M+) on CI + LARGE_FILE="$(find "${ARTIFACTS}" -name "${PATCH_NAME}" -type 'f' -size +30M)" + if [ "${IN_PROW}" -eq 0 ] && [ -n "${LARGE_FILE}" ]; then + rm "${PATCH_OUT}" + echo "WARNING: patch file was too large to persist ($(du -h "${PATCH_OUT}"))" + return 0 + fi + outName="artifacts/${PATCH_OUT#"${ARTIFACTS}"/}" + patchFile="${PROW_ARTIFACTS_BASE:-https://gcsweb.istio.io/gcs/istio-prow}/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}/${outName}" + echo "You can also try applying the patch file from the build artifacts: + +git apply <(curl -sL \"${patchFile}\") +" +} + +if [[ -n $(git status --porcelain) ]]; then + git status + git diff + echo "ERROR: Some files need to be updated, please run 'make gen' and include any changed files in your PR" + write_patch_file + exit 1 +fi diff --git a/common/scripts/copyright-banner-go.txt b/common/scripts/copyright-banner-go.txt new file mode 100644 index 000000000..ffac692d1 --- /dev/null +++ b/common/scripts/copyright-banner-go.txt @@ -0,0 +1,14 @@ +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + diff --git a/common/scripts/fix_copyright_banner.sh b/common/scripts/fix_copyright_banner.sh new file mode 100755 index 000000000..92d64e758 --- /dev/null +++ b/common/scripts/fix_copyright_banner.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +WD=$(dirname "$0") +WD=$(cd "$WD"; pwd) + +for fn in "$@"; do + if ! grep -L -q -e "Apache License, Version 2" -e "Copyright" "${fn}"; then + if [[ "${fn}" == *.go || "${fn}" == *.rs ]]; then + newfile=$(cat "${WD}/copyright-banner-go.txt" "${fn}") + echo "${newfile}" > "${fn}" + echo "Fixing license: ${fn}" + else + echo "Cannot fix license: ${fn}. Unknown file type" + fi + fi +done diff --git a/common/scripts/format_go.sh b/common/scripts/format_go.sh new file mode 100755 index 000000000..186052707 --- /dev/null +++ b/common/scripts/format_go.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +golangci-lint run --fix -c ./common/config/.golangci-format.yml diff --git a/common/scripts/gobuild.sh b/common/scripts/gobuild.sh new file mode 100755 index 000000000..25f2c8120 --- /dev/null +++ b/common/scripts/gobuild.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script builds and version stamps the output + +VERBOSE=${VERBOSE:-"0"} +V="" +if [[ "${VERBOSE}" == "1" ]];then + V="-x" + set -x +fi + +SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +OUT=${1:?"output path"} +shift + +set -e + +export BUILD_GOOS=${GOOS:-linux} +export BUILD_GOARCH=${GOARCH:-amd64} +GOBINARY=${GOBINARY:-go} +GOPKG="$GOPATH/pkg" +BUILDINFO=${BUILDINFO:-""} +STATIC=${STATIC:-1} +LDFLAGS=${LDFLAGS:--extldflags -static} +GOBUILDFLAGS=${GOBUILDFLAGS:-""} +# Split GOBUILDFLAGS by spaces into an array called GOBUILDFLAGS_ARRAY. +IFS=' ' read -r -a GOBUILDFLAGS_ARRAY <<< "$GOBUILDFLAGS" + +GCFLAGS=${GCFLAGS:-} +export CGO_ENABLED=${CGO_ENABLED:-0} + +if [[ "${STATIC}" != "1" ]];then + LDFLAGS="" +fi + +# gather buildinfo if not already provided +# For a release build BUILDINFO should be produced +# at the beginning of the build and used throughout +if [[ -z ${BUILDINFO} ]];then + BUILDINFO=$(mktemp) + "${SCRIPTPATH}/report_build_info.sh" > "${BUILDINFO}" +fi + +# BUILD LD_EXTRAFLAGS +LD_EXTRAFLAGS="" + +while read -r line; do + LD_EXTRAFLAGS="${LD_EXTRAFLAGS} -X ${line}" +done < "${BUILDINFO}" + +OPTIMIZATION_FLAGS=(-trimpath) +if [ "${DEBUG}" == "1" ]; then + OPTIMIZATION_FLAGS=() +fi + +time GOOS=${BUILD_GOOS} GOARCH=${BUILD_GOARCH} ${GOBINARY} build \ + ${V} "${GOBUILDFLAGS_ARRAY[@]}" ${GCFLAGS:+-gcflags "${GCFLAGS}"} \ + -o "${OUT}" \ + "${OPTIMIZATION_FLAGS[@]}" \ + -pkgdir="${GOPKG}/${BUILD_GOOS}_${BUILD_GOARCH}" \ + -ldflags "${LDFLAGS} ${LD_EXTRAFLAGS}" "${@}" diff --git a/common/scripts/kind_provisioner.sh b/common/scripts/kind_provisioner.sh new file mode 100644 index 000000000..4a4cb4d1e --- /dev/null +++ b/common/scripts/kind_provisioner.sh @@ -0,0 +1,438 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e +set -x + +# The purpose of this file is to unify prow/lib.sh in both istio and istio.io +# repos to avoid code duplication. + +#################################################################### +################# COMMON SECTION ############################### +#################################################################### + +# DEFAULT_KIND_IMAGE is used to set the Kubernetes version for KinD unless overridden in params to setup_kind_cluster(s) +DEFAULT_KIND_IMAGE="gcr.io/istio-testing/kind-node:v1.27.3" + +# COMMON_SCRIPTS contains the directory this file is in. +COMMON_SCRIPTS=$(dirname "${BASH_SOURCE:-$0}") + +function log() { + echo -e "$(date -u '+%Y-%m-%dT%H:%M:%S.%NZ')\t$*" +} + +function retry() { + local n=1 + local max=5 + local delay=5 + while true; do + "$@" && break + if [[ $n -lt $max ]]; then + ((n++)) + log "Command failed. Attempt $n/$max:" + sleep $delay; + else + log "The command has failed after $n attempts." >&2 + return 2 + fi + done +} + +# load_cluster_topology function reads cluster configuration topology file and +# sets up environment variables used by other functions. So this should be called +# before anything else. +# +# Note: Cluster configuration topology file specifies basic configuration of each +# KinD cluster like its name, pod and service subnets and network_id. If two cluster +# have the same network_id then they belong to the same network and their pods can +# talk to each other directly. +# +# [{ "cluster_name": "cluster1","pod_subnet": "10.10.0.0/16","svc_subnet": "10.255.10.0/24","network_id": "0" }, +# { "cluster_name": "cluster2","pod_subnet": "10.20.0.0/16","svc_subnet": "10.255.20.0/24","network_id": "0" }, +# { "cluster_name": "cluster3","pod_subnet": "10.30.0.0/16","svc_subnet": "10.255.30.0/24","network_id": "1" }] +function load_cluster_topology() { + CLUSTER_TOPOLOGY_CONFIG_FILE="${1}" + + if [[ ! -f "${CLUSTER_TOPOLOGY_CONFIG_FILE}" ]]; then + log 'cluster topology configuration file is not specified' + exit 1 + fi + + export CLUSTER_NAMES + export CLUSTER_POD_SUBNETS + export CLUSTER_SVC_SUBNETS + export CLUSTER_NETWORK_ID + + KUBE_CLUSTERS=$(jq '.[] | select(.kind == "Kubernetes" or .kind == null)' "${CLUSTER_TOPOLOGY_CONFIG_FILE}") + + while read -r value; do + CLUSTER_NAMES+=("$value") + done < <(echo "${KUBE_CLUSTERS}" | jq -r '.cluster_name // .clusterName') + + while read -r value; do + CLUSTER_POD_SUBNETS+=("$value") + done < <(echo "${KUBE_CLUSTERS}" | jq -r '.pod_subnet // .podSubnet') + + while read -r value; do + CLUSTER_SVC_SUBNETS+=("$value") + done < <(echo "${KUBE_CLUSTERS}" | jq -r '.svc_subnet // .svcSubnet') + + while read -r value; do + CLUSTER_NETWORK_ID+=("$value") + done < <(echo "${KUBE_CLUSTERS}" | jq -r '.network_id // .network') + + export NUM_CLUSTERS + NUM_CLUSTERS=$(echo "${KUBE_CLUSTERS}" | jq -s 'length') + + echo "${CLUSTER_NAMES[@]}" + echo "${CLUSTER_POD_SUBNETS[@]}" + echo "${CLUSTER_SVC_SUBNETS[@]}" + echo "${CLUSTER_NETWORK_ID[@]}" + echo "${NUM_CLUSTERS}" +} + +##################################################################### +################### SINGLE-CLUSTER SECTION ###################### +##################################################################### + +# cleanup_kind_cluster takes a single parameter NAME +# and deletes the KinD cluster with that name +function cleanup_kind_cluster() { + echo "Test exited with exit code $?." + NAME="${1}" + kind export logs --name "${NAME}" "${ARTIFACTS}/kind" -v9 || true + if [[ -z "${SKIP_CLEANUP:-}" ]]; then + echo "Cleaning up kind cluster" + kind delete cluster --name "${NAME}" -v9 || true + fi +} + +# check_default_cluster_yaml checks the presence of default cluster YAML +# It returns 1 if it is not present +function check_default_cluster_yaml() { + if [[ -z "${DEFAULT_CLUSTER_YAML}" ]]; then + echo 'DEFAULT_CLUSTER_YAML file must be specified. Exiting...' + return 1 + fi +} + +function setup_kind_cluster_retry() { + retry setup_kind_cluster "$@" +} + +# setup_kind_cluster creates new KinD cluster with given name, image and configuration +# 1. NAME: Name of the Kind cluster (optional) +# 2. IMAGE: Node image used by KinD (optional) +# 3. CONFIG: KinD cluster configuration YAML file. If not specified then DEFAULT_CLUSTER_YAML is used +# 4. NOMETALBINSTALL: Dont install matllb if set. +# This function returns 0 when everything goes well, or 1 otherwise +# If Kind cluster was already created then it would be cleaned up in case of errors +function setup_kind_cluster() { + NAME="${1:-istio-testing}" + IMAGE="${2:-"${DEFAULT_KIND_IMAGE}"}" + CONFIG="${3:-}" + NOMETALBINSTALL="${4:-}" + CLEANUP="${5:-true}" + + check_default_cluster_yaml + + # Delete any previous KinD cluster + echo "Deleting previous KinD cluster with name=${NAME}" + if ! (kind delete cluster --name="${NAME}" -v9) > /dev/null; then + echo "No existing kind cluster with name ${NAME}. Continue..." + fi + + # explicitly disable shellcheck since we actually want $NAME to expand now + # shellcheck disable=SC2064 + if [[ "${CLEANUP}" == "true" ]]; then + trap "cleanup_kind_cluster ${NAME}" EXIT + fi + + # If config not explicitly set, then use defaults + if [[ -z "${CONFIG}" ]]; then + # Kubernetes 1.15+ + CONFIG=${DEFAULT_CLUSTER_YAML} + # Configure the cluster IP Family only for default configs + if [ "${IP_FAMILY}" != "ipv4" ]; then + grep "ipFamily: ${IP_FAMILY}" "${CONFIG}" || \ + cat <> "${CONFIG}" +networking: + ipFamily: ${IP_FAMILY} +EOF + fi + fi + + # Create KinD cluster + if ! (kind create cluster --name="${NAME}" --config "${CONFIG}" -v4 --retain --image "${IMAGE}" --wait=180s); then + echo "Could not setup KinD environment. Something wrong with KinD setup. Exporting logs." + return 9 + fi + # Workaround kind issue causing taints to not be removed in 1.24 + kubectl taint nodes "${NAME}"-control-plane node-role.kubernetes.io/control-plane- || true + + # If metrics server configuration directory is specified then deploy in + # the cluster just created + if [[ -n ${METRICS_SERVER_CONFIG_DIR} ]]; then + retry kubectl apply -f "${METRICS_SERVER_CONFIG_DIR}" + fi + + # Install Metallb if not set to install explicitly + if [[ -z "${NOMETALBINSTALL}" ]]; then + retry install_metallb "" + fi + + # IPv6 clusters need some CoreDNS changes in order to work in CI: + # Istio CI doesn't offer IPv6 connectivity, so CoreDNS should be configured + # to work in an offline environment: + # https://github.com/coredns/coredns/issues/2494#issuecomment-457215452 + # CoreDNS should handle those domains and answer with NXDOMAIN instead of SERVFAIL + # otherwise pods stops trying to resolve the domain. + if [ "${IP_FAMILY}" = "ipv6" ] || [ "${IP_FAMILY}" = "dual" ]; then + # Get the current config + original_coredns=$(kubectl get -oyaml -n=kube-system configmap/coredns) + echo "Original CoreDNS config:" + echo "${original_coredns}" + # Patch it + fixed_coredns=$( + printf '%s' "${original_coredns}" | sed \ + -e 's/^.*kubernetes cluster\.local/& internal/' \ + -e '/^.*upstream$/d' \ + -e '/^.*fallthrough.*$/d' \ + -e '/^.*forward . \/etc\/resolv.conf$/d' \ + -e '/^.*loop$/d' \ + ) + echo "Patched CoreDNS config:" + echo "${fixed_coredns}" + printf '%s' "${fixed_coredns}" | kubectl apply -f - + fi +} + +############################################################################### +#################### MULTICLUSTER SECTION ############################### +############################################################################### + +# Cleans up the clusters created by setup_kind_clusters +# It expects CLUSTER_NAMES to be present which means that +# load_cluster_topology must be called before invoking it +function cleanup_kind_clusters() { + echo "Test exited with exit code $?." + for c in "${CLUSTER_NAMES[@]}"; do + cleanup_kind_cluster "${c}" + done +} + +# setup_kind_clusters sets up a given number of kind clusters with given topology +# as specified in cluster topology configuration file. +# 1. IMAGE = docker image used as node by KinD +# 2. IP_FAMILY = either ipv4 or ipv6 +# +# NOTE: Please call load_cluster_topology before calling this method as it expects +# cluster topology information to be loaded in advance +function setup_kind_clusters() { + IMAGE="${1:-"${DEFAULT_KIND_IMAGE}"}" + KUBECONFIG_DIR="${ARTIFACTS:-$(mktemp -d)}/kubeconfig" + IP_FAMILY="${2:-ipv4}" + + check_default_cluster_yaml + + # Trap replaces any previous trap's, so we need to explicitly cleanup clusters here + trap cleanup_kind_clusters EXIT + + function deploy_kind() { + IDX="${1}" + CLUSTER_NAME="${CLUSTER_NAMES[$IDX]}" + CLUSTER_POD_SUBNET="${CLUSTER_POD_SUBNETS[$IDX]}" + CLUSTER_SVC_SUBNET="${CLUSTER_SVC_SUBNETS[$IDX]}" + CLUSTER_YAML="${ARTIFACTS}/config-${CLUSTER_NAME}.yaml" + if [ ! -f "${CLUSTER_YAML}" ]; then + cp "${DEFAULT_CLUSTER_YAML}" "${CLUSTER_YAML}" + cat <> "${CLUSTER_YAML}" +networking: + podSubnet: ${CLUSTER_POD_SUBNET} + serviceSubnet: ${CLUSTER_SVC_SUBNET} +EOF + fi + + CLUSTER_KUBECONFIG="${KUBECONFIG_DIR}/${CLUSTER_NAME}" + + # Create the clusters. + KUBECONFIG="${CLUSTER_KUBECONFIG}" setup_kind_cluster "${CLUSTER_NAME}" "${IMAGE}" "${CLUSTER_YAML}" "true" "false" + + # Kind currently supports getting a kubeconfig for internal or external usage. To simplify our tests, + # its much simpler if we have a single kubeconfig that can be used internally and externally. + # To do this, we can replace the server with the IP address of the docker container + # https://github.com/kubernetes-sigs/kind/issues/1558 tracks this upstream + CONTAINER_IP=$(docker inspect "${CLUSTER_NAME}-control-plane" --format "{{ .NetworkSettings.Networks.kind.IPAddress }}") + n=0 + until [ $n -ge 10 ]; do + n=$((n+1)) + kind get kubeconfig --name "${CLUSTER_NAME}" --internal | \ + sed "s/${CLUSTER_NAME}-control-plane/${CONTAINER_IP}/g" > "${CLUSTER_KUBECONFIG}" + [ -s "${CLUSTER_KUBECONFIG}" ] && break + sleep 3 + done + + # Enable core dumps + retry docker exec "${CLUSTER_NAME}"-control-plane bash -c "sysctl -w kernel.core_pattern=/var/lib/istio/data/core.proxy && ulimit -c unlimited" + } + + # Now deploy the specified number of KinD clusters and + # wait till they are provisioned successfully. + declare -a DEPLOY_KIND_JOBS + for i in "${!CLUSTER_NAMES[@]}"; do + deploy_kind "${i}" & DEPLOY_KIND_JOBS+=("${!}") + done + + for pid in "${DEPLOY_KIND_JOBS[@]}"; do + wait "${pid}" || exit 1 + done + + # Install MetalLB for LoadBalancer support. Must be done synchronously since METALLB_IPS is shared. + # and keep track of the list of Kubeconfig files that will be exported later + export KUBECONFIGS + for CLUSTER_NAME in "${CLUSTER_NAMES[@]}"; do + KUBECONFIG_FILE="${KUBECONFIG_DIR}/${CLUSTER_NAME}" + if [[ ${NUM_CLUSTERS} -gt 1 ]]; then + retry install_metallb "${KUBECONFIG_FILE}" + fi + KUBECONFIGS+=("${KUBECONFIG_FILE}") + done + + ITER_END=$((NUM_CLUSTERS-1)) + for i in $(seq 0 "$ITER_END"); do + for j in $(seq 0 "$ITER_END"); do + if [[ "${j}" -gt "${i}" ]]; then + NETWORK_ID_I="${CLUSTER_NETWORK_ID[i]}" + NETWORK_ID_J="${CLUSTER_NETWORK_ID[j]}" + if [[ "$NETWORK_ID_I" == "$NETWORK_ID_J" ]]; then + POD_TO_POD_AND_SERVICE_CONNECTIVITY=1 + else + POD_TO_POD_AND_SERVICE_CONNECTIVITY=0 + fi + connect_kind_clusters \ + "${CLUSTER_NAMES[i]}" "${KUBECONFIGS[i]}" \ + "${CLUSTER_NAMES[j]}" "${KUBECONFIGS[j]}" \ + "${POD_TO_POD_AND_SERVICE_CONNECTIVITY}" + fi + done + done +} + +function connect_kind_clusters() { + C1="${1}" + C1_KUBECONFIG="${2}" + C2="${3}" + C2_KUBECONFIG="${4}" + POD_TO_POD_AND_SERVICE_CONNECTIVITY="${5}" + + C1_NODE="${C1}-control-plane" + C2_NODE="${C2}-control-plane" + C1_DOCKER_IP=$(docker inspect -f "{{ .NetworkSettings.Networks.kind.IPAddress }}" "${C1_NODE}") + C2_DOCKER_IP=$(docker inspect -f "{{ .NetworkSettings.Networks.kind.IPAddress }}" "${C2_NODE}") + if [ "${POD_TO_POD_AND_SERVICE_CONNECTIVITY}" -eq 1 ]; then + # Set up routing rules for inter-cluster direct pod to pod & service communication + C1_POD_CIDR=$(KUBECONFIG="${C1_KUBECONFIG}" kubectl get node -ojsonpath='{.items[0].spec.podCIDR}') + C2_POD_CIDR=$(KUBECONFIG="${C2_KUBECONFIG}" kubectl get node -ojsonpath='{.items[0].spec.podCIDR}') + C1_SVC_CIDR=$(KUBECONFIG="${C1_KUBECONFIG}" kubectl cluster-info dump | sed -n 's/^.*--service-cluster-ip-range=\([^"]*\).*$/\1/p' | head -n 1) + C2_SVC_CIDR=$(KUBECONFIG="${C2_KUBECONFIG}" kubectl cluster-info dump | sed -n 's/^.*--service-cluster-ip-range=\([^"]*\).*$/\1/p' | head -n 1) + docker exec "${C1_NODE}" ip route add "${C2_POD_CIDR}" via "${C2_DOCKER_IP}" + docker exec "${C1_NODE}" ip route add "${C2_SVC_CIDR}" via "${C2_DOCKER_IP}" + docker exec "${C2_NODE}" ip route add "${C1_POD_CIDR}" via "${C1_DOCKER_IP}" + docker exec "${C2_NODE}" ip route add "${C1_SVC_CIDR}" via "${C1_DOCKER_IP}" + fi +} + +function install_metallb() { + KUBECONFIG="${1}" + kubectl apply --kubeconfig="$KUBECONFIG" -f "${COMMON_SCRIPTS}/metallb.yaml" + kubectl create --kubeconfig="$KUBECONFIG" secret generic -n metallb-system memberlist --from-literal=secretkey="$(openssl rand -base64 128)" + + if [ -z "${METALLB_IPS4+x}" ]; then + # Take IPs from the end of the docker kind network subnet to use for MetalLB IPs + DOCKER_KIND_SUBNET="$(docker inspect kind | jq '.[0].IPAM.Config[0].Subnet' -r)" + METALLB_IPS4=() + while read -r ip; do + METALLB_IPS4+=("$ip") + done < <(cidr_to_ips "$DOCKER_KIND_SUBNET" | tail -n 100) + METALLB_IPS6=() + if [[ "$(docker inspect kind | jq '.[0].IPAM.Config | length' -r)" == 2 ]]; then + # Two configs? Must be dual stack. + DOCKER_KIND_SUBNET="$(docker inspect kind | jq '.[0].IPAM.Config[1].Subnet' -r)" + while read -r ip; do + METALLB_IPS6+=("$ip") + done < <(cidr_to_ips "$DOCKER_KIND_SUBNET" | tail -n 100) + fi + fi + + # Give this cluster of those IPs + RANGE="[" + for i in {0..19}; do + RANGE+="${METALLB_IPS4[1]}," + METALLB_IPS4=("${METALLB_IPS4[@]:1}") + if [[ "${#METALLB_IPS6[@]}" != 0 ]]; then + RANGE+="${METALLB_IPS6[1]}," + METALLB_IPS6=("${METALLB_IPS6[@]:1}") + fi + done + RANGE="${RANGE%?}]" + + echo 'apiVersion: v1 +kind: ConfigMap +metadata: + namespace: metallb-system + name: config +data: + config: | + address-pools: + - name: default + protocol: layer2 + addresses: '"$RANGE" | kubectl apply --kubeconfig="$KUBECONFIG" -f - +} + +function cidr_to_ips() { + CIDR="$1" + # cidr_to_ips returns a list of single IPs from a CIDR. We skip 1000 (since they are likely to be allocated + # already to other services), then pick the next 100. + python3 - < 2000: + start, end = 1000, 2000 + +[print(str(ip) + "/" + str(ip.max_prefixlen)) for ip in islice(ip_network('$CIDR').hosts(), start, end)] +EOF +} + +function ips_to_cidrs() { + IP_RANGE_START="$1" + IP_RANGE_END="$2" + python3 - < /dev/null); then + if [[ -z "${IGNORE_DIRTY_TREE}" ]] && [[ -n "$(git status --porcelain 2>/dev/null)" ]]; then + BUILD_GIT_REVISION=${BUILD_GIT_REVISION}"-dirty" + fi +else + BUILD_GIT_REVISION=unknown +fi + +# Check for local changes +tree_status="Clean" +if [[ -z "${IGNORE_DIRTY_TREE}" ]] && ! git diff-index --quiet HEAD --; then + tree_status="Modified" +fi + +GIT_DESCRIBE_TAG=$(git describe --tags --always) +HUB=${HUB:-"docker.io/istio"} + +# used by common/scripts/gobuild.sh +echo "istio.io/istio/pkg/version.buildVersion=${VERSION:-$BUILD_GIT_REVISION}" +echo "istio.io/istio/pkg/version.buildGitRevision=${BUILD_GIT_REVISION}" +echo "istio.io/istio/pkg/version.buildStatus=${tree_status}" +echo "istio.io/istio/pkg/version.buildTag=${GIT_DESCRIBE_TAG}" +echo "istio.io/istio/pkg/version.buildHub=${HUB}" +echo "istio.io/istio/pkg/version.buildOS=${BUILD_GOOS}" +echo "istio.io/istio/pkg/version.buildArch=${BUILD_GOARCH}" diff --git a/common/scripts/run.sh b/common/scripts/run.sh new file mode 100755 index 000000000..7a7a4d5ba --- /dev/null +++ b/common/scripts/run.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +WD=$(dirname "$0") +WD=$(cd "$WD"; pwd) + +export FOR_BUILD_CONTAINER=1 +# shellcheck disable=SC1090,SC1091 +source "${WD}/setup_env.sh" + + +MOUNT_SOURCE="${MOUNT_SOURCE:-${PWD}}" +MOUNT_DEST="${MOUNT_DEST:-/work}" + +read -ra DOCKER_RUN_OPTIONS <<< "${DOCKER_RUN_OPTIONS:-}" + +[[ -t 1 ]] && DOCKER_RUN_OPTIONS+=("-it") +[[ ${UID} -ne 0 ]] && DOCKER_RUN_OPTIONS+=(-u "${UID}:${DOCKER_GID}") + +# $CONTAINER_OPTIONS becomes an empty arg when quoted, so SC2086 is disabled for the +# following command only +# shellcheck disable=SC2086 +"${CONTAINER_CLI}" run \ + --rm \ + "${DOCKER_RUN_OPTIONS[@]}" \ + --init \ + --sig-proxy=true \ + ${DOCKER_SOCKET_MOUNT:--v /var/run/docker.sock:/var/run/docker.sock} \ + $CONTAINER_OPTIONS \ + --env-file <(env | grep -v ${ENV_BLOCKLIST}) \ + -e IN_BUILD_CONTAINER=1 \ + -e TZ="${TIMEZONE:-$TZ}" \ + --mount "type=bind,source=${MOUNT_SOURCE},destination=/work" \ + --mount "type=volume,source=go,destination=/go" \ + --mount "type=volume,source=gocache,destination=/gocache" \ + --mount "type=volume,source=cache,destination=/home/.cache" \ + --mount "type=volume,source=crates,destination=/home/.cargo/registry" \ + --mount "type=volume,source=git-crates,destination=/home/.cargo/git" \ + ${CONDITIONAL_HOST_MOUNTS} \ + -w "${MOUNT_DEST}" "${IMG}" "$@" diff --git a/common/scripts/setup_env.sh b/common/scripts/setup_env.sh new file mode 100755 index 000000000..6e2f1bd21 --- /dev/null +++ b/common/scripts/setup_env.sh @@ -0,0 +1,233 @@ +#!/bin/bash +# shellcheck disable=SC2034 + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# https://stackoverflow.com/questions/59895/how-can-i-get-the-source-directory-of-a-bash-script-from-within-the-script-itsel +# Note: the normal way we use in other scripts in Istio do not work when `source`d, which is why we use this approach +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +REPO_ROOT="$(dirname "$(dirname "${SCRIPT_DIR}")")" + +LOCAL_ARCH=$(uname -m) + +# Pass environment set target architecture to build system +if [[ ${TARGET_ARCH} ]]; then + # Target explicitly set + : +elif [[ ${LOCAL_ARCH} == x86_64 ]]; then + TARGET_ARCH=amd64 +elif [[ ${LOCAL_ARCH} == armv8* ]]; then + TARGET_ARCH=arm64 +elif [[ ${LOCAL_ARCH} == arm64* ]]; then + TARGET_ARCH=arm64 +elif [[ ${LOCAL_ARCH} == aarch64* ]]; then + TARGET_ARCH=arm64 +elif [[ ${LOCAL_ARCH} == armv* ]]; then + TARGET_ARCH=arm +elif [[ ${LOCAL_ARCH} == s390x ]]; then + TARGET_ARCH=s390x +elif [[ ${LOCAL_ARCH} == ppc64le ]]; then + TARGET_ARCH=ppc64le +else + echo "This system's architecture, ${LOCAL_ARCH}, isn't supported" + exit 1 +fi + +LOCAL_OS=$(uname) + +# Pass environment set target operating-system to build system +if [[ ${TARGET_OS} ]]; then + # Target explicitly set + : +elif [[ $LOCAL_OS == Linux ]]; then + TARGET_OS=linux + readlink_flags="-f" +elif [[ $LOCAL_OS == Darwin ]]; then + TARGET_OS=darwin + readlink_flags="" +else + echo "This system's OS, $LOCAL_OS, isn't supported" + exit 1 +fi + +# Build image to use +TOOLS_REGISTRY_PROVIDER=${TOOLS_REGISTRY_PROVIDER:-gcr.io} +PROJECT_ID=${PROJECT_ID:-istio-testing} +if [[ "${IMAGE_VERSION:-}" == "" ]]; then + IMAGE_VERSION=master-d5b39ac8e423b269748c9c1e9692a853db960fc9 +fi +if [[ "${IMAGE_NAME:-}" == "" ]]; then + IMAGE_NAME=build-tools +fi + +DOCKER_GID="${DOCKER_GID:-$(grep '^docker:' /etc/group | cut -f3 -d:)}" + +TIMEZONE=$(readlink "$readlink_flags" /etc/localtime | sed -e 's/^.*zoneinfo\///') + +TARGET_OUT="${TARGET_OUT:-$(pwd)/out/${TARGET_OS}_${TARGET_ARCH}}" +TARGET_OUT_LINUX="${TARGET_OUT_LINUX:-$(pwd)/out/linux_${TARGET_ARCH}}" + +CONTAINER_TARGET_OUT="${CONTAINER_TARGET_OUT:-/work/out/${TARGET_OS}_${TARGET_ARCH}}" +CONTAINER_TARGET_OUT_LINUX="${CONTAINER_TARGET_OUT_LINUX:-/work/out/linux_${TARGET_ARCH}}" + +IMG="${IMG:-${TOOLS_REGISTRY_PROVIDER}/${PROJECT_ID}/${IMAGE_NAME}:${IMAGE_VERSION}}" + +CONTAINER_CLI="${CONTAINER_CLI:-docker}" + +ENV_BLOCKLIST="${ENV_BLOCKLIST:-^_\|^PATH=\|^GOPATH=\|^GOROOT=\|^SHELL=\|^EDITOR=\|^TMUX=\|^USER=\|^HOME=\|^PWD=\|^TERM=\|^RUBY_\|^GEM_\|^rvm_\|^SSH=\|^TMPDIR=\|^CC=\|^CXX=\|^MAKEFILE_LIST=}" + +# Remove functions from the list of exported variables, they mess up with the `env` command. +for f in $(declare -F -x | cut -d ' ' -f 3); +do + unset -f "${f}" +done + +# Set conditional host mounts +CONDITIONAL_HOST_MOUNTS="${CONDITIONAL_HOST_MOUNTS:-} " +container_kubeconfig='' + +# docker conditional host mount (needed for make docker push) +if [[ -d "${HOME}/.docker" ]]; then + CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${HOME}/.docker,destination=/config/.docker,readonly " +fi + +# gcloud conditional host mount (needed for docker push with the gcloud auth configure-docker) +if [[ -d "${HOME}/.config/gcloud" ]]; then + CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${HOME}/.config/gcloud,destination=/config/.config/gcloud,readonly " +fi + +# gitconfig conditional host mount (needed for git commands inside container) +if [[ -f "${HOME}/.gitconfig" ]]; then + CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${HOME}/.gitconfig,destination=/home/.gitconfig,readonly " +fi + +# .netrc conditional host mount (needed for git commands inside container) +if [[ -f "${HOME}/.netrc" ]]; then + CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${HOME}/.netrc,destination=/home/.netrc,readonly " +fi + +# echo ${CONDITIONAL_HOST_MOUNTS} + +# This function checks if the file exists. If it does, it creates a randomly named host location +# for the file, adds it to the host KUBECONFIG, and creates a mount for it. +add_KUBECONFIG_if_exists () { + if [[ -f "$1" ]]; then + kubeconfig_random="$(od -vAn -N4 -tx /dev/random | tr -d '[:space:]' | cut -c1-8)" + container_kubeconfig+="/config/${kubeconfig_random}:" + CONDITIONAL_HOST_MOUNTS+="--mount type=bind,source=${1},destination=/config/${kubeconfig_random},readonly " + fi +} + +# This function is designed for maximum compatibility with various platforms. This runs on +# any Mac or Linux platform with bash 4.2+. Please take care not to modify this function +# without testing properly. +# +# This function will properly handle any type of path including those with spaces using the +# loading pattern specified by *kubectl config*. +# +# testcase: "a:b c:d" +# testcase: "a b:c d:e f" +# testcase: "a b:c:d e" +parse_KUBECONFIG () { +TMPDIR="" +if [[ "$1" =~ ([^:]*):(.*) ]]; then + while true; do + rematch=${BASH_REMATCH[1]} + add_KUBECONFIG_if_exists "$rematch" + remainder="${BASH_REMATCH[2]}" + if [[ ! "$remainder" =~ ([^:]*):(.*) ]]; then + if [[ -n "$remainder" ]]; then + add_KUBECONFIG_if_exists "$remainder" + break + fi + fi + done +else + add_KUBECONFIG_if_exists "$1" +fi +} + +KUBECONFIG=${KUBECONFIG:="$HOME/.kube/config"} +parse_KUBECONFIG "${KUBECONFIG}" +if [[ "${FOR_BUILD_CONTAINER:-0}" -eq "1" ]]; then + KUBECONFIG="${container_kubeconfig%?}" +fi + +# LOCAL_OUT should point to architecture where we are currently running versus the desired. +# This is used when we need to run a build artifact during tests or later as part of another +# target. +if [[ "${FOR_BUILD_CONTAINER:-0}" -eq "1" ]]; then + # Override variables with container specific + TARGET_OUT=${CONTAINER_TARGET_OUT} + TARGET_OUT_LINUX=${CONTAINER_TARGET_OUT_LINUX} + REPO_ROOT=/work + LOCAL_OUT="${TARGET_OUT_LINUX}" +else + LOCAL_OUT="${TARGET_OUT}" +fi + +go_os_arch=${LOCAL_OUT##*/} +# Golang OS/Arch format +LOCAL_GO_OS=${go_os_arch%_*} +LOCAL_GO_ARCH=${go_os_arch##*_} + +BUILD_WITH_CONTAINER=0 + +VARS=( + CONTAINER_TARGET_OUT + CONTAINER_TARGET_OUT_LINUX + TARGET_OUT + TARGET_OUT_LINUX + LOCAL_GO_OS + LOCAL_GO_ARCH + LOCAL_OUT + LOCAL_OS + TARGET_OS + LOCAL_ARCH + TARGET_ARCH + TIMEZONE + KUBECONFIG + CONDITIONAL_HOST_MOUNTS + ENV_BLOCKLIST + CONTAINER_CLI + DOCKER_GID + IMG + IMAGE_NAME + IMAGE_VERSION + REPO_ROOT + BUILD_WITH_CONTAINER +) + +# For non container build, we need to write env to file +if [[ "${1}" == "envfile" ]]; then + # ! does a variable-variable https://stackoverflow.com/a/10757531/374797 + for var in "${VARS[@]}"; do + echo "${var}"="${!var}" + done +else + for var in "${VARS[@]}"; do + # shellcheck disable=SC2163 + export "${var}" + done +fi diff --git a/common/scripts/tracing.sh b/common/scripts/tracing.sh new file mode 100755 index 000000000..7ac41e52b --- /dev/null +++ b/common/scripts/tracing.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY +# +# The original version of this file is located in the https://github.com/istio/common-files repo. +# If you're looking at this file in a different repo and want to make a change, please go to the +# common-files repo, make the change there and check it in. Then come back to this repo and run +# "make update-common". + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: tracing::extract_prow_trace. +# If running in a prow job, this sets the parent trace to the same value Prow tracing will use, as defined in https://github.com/kubernetes/test-infra/issues/30010 +function tracing::extract_prow_trace() { + if [[ "${PROW_JOB_ID:-}" != "" ]]; then + local trace + trace="$(<<< "$PROW_JOB_ID" tr -d '\-')" + local span + span="${trace:0:16}" + export TRACEPARENT="01-${trace}-${span}-00" + fi +} + +function _genattrs() { + # No upstream standard, so copy from https://github.com/jenkinsci/opentelemetry-plugin/blob/master/docs/job-traces.md + if [[ -n "${PULL_NUMBER:=}" ]] + then + # Presubmit + url="https://prow.istio.io/view/gs/istio-prow/pr-logs/pull/${REPO_OWNER}_${REPO_NAME}/${PULL_NUMBER}/${JOB_NAME}/${BUILD_ID}," + else + # Postsubmit or periodic + url="https://prow.istio.io/view/gs/istio-prow/pr-logs/${JOB_NAME}/${BUILD_ID}," + fi + # Use printf instead of echo to avoid spaces between args + printf '%s' "ci.pipeline.id=${JOB_NAME},"\ + "ci.pipeline.type=${JOB_TYPE},"\ + "ci.pipeline.run.url=${url}"\ + "ci.pipeline.run.number=${BUILD_ID},"\ + "ci.pipeline.run.id=${PROW_JOB_ID},"\ + "ci.pipeline.run.repo=${REPO_OWNER:-unknown}/${REPO_NAME:-unknown},"\ + "ci.pipeline.run.base=${PULL_BASE_REF:-none},"\ + "ci.pipeline.run.pull_number=${PULL_NUMBER:-none},"\ + "ci.pipeline.run.pull_sha=${PULL_PULL_SHA:-${PULL_BASE_SHA:-none}}" +} + +# Usage: tracing::run [command ...] +function tracing::run() { + # If not running in a prow job or otel-cli is not available (e.g. build system without otel-cli) just run the command + if [ -z "${JOB_NAME:-}" ] || ! command -v otel-cli &> /dev/null + then + "${@:2}" + return "$?" + fi + + # Disable execution tracing to avoid noise + { [[ $- = *x* ]] && was_execution_trace=1 || was_execution_trace=0; } 2>/dev/null + { set +x; } 2>/dev/null + # Throughout, "local" usage is critical to avoid nested calls overwriting things + local start + start="$(date -u +%s.%N)" + # First, get a trace and span ID. We need to get one now so we can propagate it to the child + # Get trace ID from TRACEPARENT, if present + local tid + tid="$(<<<"${TRACEPARENT:-}" cut -d- -f2)" + tid="${tid:-"$(tr -dc 'a-f0-9' < /dev/urandom | head -c 32)"}" + # Always generate a new span ID + local sid + sid="$(tr -dc 'a-f0-9' < /dev/urandom | head -c 16)" + + # Execute the command they wanted with the propagation through TRACEPARENT + if [[ $was_execution_trace == 1 ]]; then + { set -x; } 2>/dev/null + fi + + TRACEPARENT="00-${tid}-${sid}-01" "${@:2}" + local result="$?" + { set +x; } 2>/dev/null + + local end + end="$(date -u +%s.%N)" + + # Now report this span. We override the IDs to the ones we set before. + otel-cli span \ + --service "${BASH_SOURCE[-1]}" \ + --name "$1" \ + --start "$start" \ + --end "$end" \ + --force-trace-id "$tid" \ + --force-span-id "$sid" \ + --attrs "$(_genattrs)" + if [[ $was_execution_trace == 1 ]]; then + { set -x; } 2>/dev/null + fi + return "$result" +} + +# Usage: tracing::decorate +# Automatically makes a function traced. +function tracing::decorate() { +eval "\ +function $1() { +_$(typeset -f "$1") +tracing::run '$1' _$1 +} +" +} diff --git a/out/.env b/out/.env new file mode 100644 index 000000000..b33ee58c9 --- /dev/null +++ b/out/.env @@ -0,0 +1,22 @@ +CONTAINER_TARGET_OUT=/work/out/linux_amd64 +CONTAINER_TARGET_OUT_LINUX=/work/out/linux_amd64 +TARGET_OUT=/home/jwendell/src/maistra/istio-operator-3.0/out/linux_amd64 +TARGET_OUT_LINUX=/home/jwendell/src/maistra/istio-operator-3.0/out/linux_amd64 +LOCAL_GO_OS=linux +LOCAL_GO_ARCH=amd64 +LOCAL_OUT=/home/jwendell/src/maistra/istio-operator-3.0/out/linux_amd64 +LOCAL_OS=Linux +TARGET_OS=linux +LOCAL_ARCH=x86_64 +TARGET_ARCH=amd64 +TIMEZONE=America/New_York +KUBECONFIG=/home/jwendell/.kube/config +CONDITIONAL_HOST_MOUNTS= --mount type=bind,source=/home/jwendell/.docker,destination=/config/.docker,readonly --mount type=bind,source=/home/jwendell/.config/gcloud,destination=/config/.config/gcloud,readonly --mount type=bind,source=/home/jwendell/.gitconfig,destination=/home/.gitconfig,readonly --mount type=bind,source=/home/jwendell/.kube/config,destination=/config/1a0db0e3,readonly +ENV_BLOCKLIST=^_\|^PATH=\|^GOPATH=\|^GOROOT=\|^SHELL=\|^EDITOR=\|^TMUX=\|^USER=\|^HOME=\|^PWD=\|^TERM=\|^RUBY_\|^GEM_\|^rvm_\|^SSH=\|^TMPDIR=\|^CC=\|^CXX=\|^MAKEFILE_LIST= +CONTAINER_CLI=docker +DOCKER_GID=975 +IMG=gcr.io/istio-testing/build-tools:master-d5b39ac8e423b269748c9c1e9692a853db960fc9 +IMAGE_NAME=build-tools +IMAGE_VERSION=master-d5b39ac8e423b269748c9c1e9692a853db960fc9 +REPO_ROOT=/home/jwendell/src/maistra/istio-operator-3.0 +BUILD_WITH_CONTAINER=0 diff --git a/tools/commonfiles-postprocess.sh b/tools/commonfiles-postprocess.sh new file mode 100755 index 000000000..71a276bbb --- /dev/null +++ b/tools/commonfiles-postprocess.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# Copyright Istio Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# shellcheck disable=SC2016 +yq ea '. as $item ireduce ({}; . *+ $item )' ./common/config/.golangci.yml ./tools/golangci-override.yaml > ./common/config/.golangci.yml.tmp +mv ./common/config/.golangci.yml.tmp ./common/config/.golangci.yml diff --git a/tools/golangci-override.yaml b/tools/golangci-override.yaml new file mode 100644 index 000000000..5ff5de952 --- /dev/null +++ b/tools/golangci-override.yaml @@ -0,0 +1,10 @@ +issues: + exclude-rules: + - linters: + - revive + text: "struct-tag: unknown option 'inline' in JSON tag" + + # Don't check *_types.go files for lines that are too long (the linter complains about +kubebuilder:printcolumn lines) + - path: _types.go$ + linters: + - lll