From 72304954157d16c1ff0e20e2737ac1282bf2c148 Mon Sep 17 00:00:00 2001 From: Dilan Bellinghoven Date: Fri, 7 Jun 2019 16:48:34 -0400 Subject: [PATCH] feat: Improve pipeline --- .golangci.toml | 54 +++++++++ Makefile | 12 +- ci/lint/errcheck-exclude.txt | 4 + ci/pipeline.yml | 108 ++++++++++++++---- ci/release/build-release.sh | 55 +++++++++ .../publish-release.sh} | 14 +-- ci/release/requirements.txt | 2 + ci/release/version.py | 56 +++++++++ ci/run-tests.sh | 29 ----- ci/tasks/build-release.yml | 10 +- ci/tasks/publish-release.yml | 9 ++ .../{test-pull-request.yml => validate.yml} | 4 +- 12 files changed, 281 insertions(+), 76 deletions(-) create mode 100644 .golangci.toml create mode 100644 ci/lint/errcheck-exclude.txt create mode 100755 ci/release/build-release.sh rename ci/{build-release.sh => release/publish-release.sh} (73%) create mode 100644 ci/release/requirements.txt create mode 100644 ci/release/version.py delete mode 100755 ci/run-tests.sh create mode 100644 ci/tasks/publish-release.yml rename ci/tasks/{test-pull-request.yml => validate.yml} (51%) diff --git a/.golangci.toml b/.golangci.toml new file mode 100644 index 0000000..ed69950 --- /dev/null +++ b/.golangci.toml @@ -0,0 +1,54 @@ +[run] + deadline = "5m" + +[issues] +exclude = [ + # golint: False positives for multi-file packages + 'should have a package comment', + # gosec: Duplicate of errcheck + 'G104: Errors unhandled', +] +exclude-rules = [ + { + path = '_test\.go', + linters = [ + 'dupl', + 'goconst', + 'gocyclo', + 'scopelint', + 'lll', + ] + }, +] +exclude-use-default = false + +[linters] +enable-all = true +disable = [ + "gochecknoglobals", # There are valid use cases for globals + "maligned", # Readability is more important than saving bytes +] + +[linters-settings] + + [linters-settings.errcheck] + exclude = "./ci/lint/errcheck-exclude.txt" + + [linters-settings.goconst] + min-len = 0 + min-occurrences = 3 + + [linters-settings.gocyclo] + min-complexity = 10 + + [linters-settings.golint] + min-confidence = 0.0 + + [linters-settings.govet] + check-shadowing = true + + [linters-settings.misspell] + locale = "US" + + [linters-settings.nakedret] + max-func-lines = 0 diff --git a/Makefile b/Makefile index 1c21b2f..1cd63ff 100644 --- a/Makefile +++ b/Makefile @@ -24,10 +24,6 @@ LOCAL_BINARY=bin/$(BINARY_NAME) all: build -update_deps: - @GO111MODULE=on go mod tidy -v && go mod vendor -v -.PHONY: update_deps - docker: Dockerfile scripts/docker-build.sh .PHONY: docker @@ -40,10 +36,6 @@ $(LOCAL_BINARY): $(SOURCES) @sh -c "'./scripts/build-binary.sh' '$(shell git describe --tags --abbrev=0)' '$(shell git rev-parse --short HEAD)' '$(REPO)'" @echo "==> Done. Your binary can be found at bin/go-elasticsearch-alerts." -test: - @GO111MODULE=on go test -v -cover ./... -.PHONY: test - git_chglog_check: if [ -z "$(shell which git-chglog)" ]; then \ GOPATH=$(shell pwd) PATH=$$PATH:$(shell pwd)/bin go get -u -v github.com/git-chglog/git-chglog/cmd/git-chglog && GOPATH=$(shell pwd) PATH=$$PATH:$(shell pwd)/bin git-chglog --version; \ @@ -79,7 +71,9 @@ set_pipeline: check_fly --config ci/pipeline.yml \ --pipeline $(CONCOURSE_PIPELINE) \ --non-interactive \ - -v github-repo="$$(git config remote.origin.url)" + -v github-repo="$$(git config remote.origin.url)" \ + -v github-actor="Dilan Bellinghoven" \ + -v github-email="dbellinghoven@morningconsult.com" $(FLY) --target mci-ci-oss unpause-pipeline \ --pipeline $(CONCOURSE_PIPELINE) diff --git a/ci/lint/errcheck-exclude.txt b/ci/lint/errcheck-exclude.txt new file mode 100644 index 0000000..cef576e --- /dev/null +++ b/ci/lint/errcheck-exclude.txt @@ -0,0 +1,4 @@ +(io.Closer).Close +(net/http.ResponseWriter).Write +os.Setenv +(github.com/hashicorp/consul/api.Lock).Unlock \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 5eed140..22f4866 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -22,26 +22,40 @@ resource_types: # # ==================================== resources: -- name: test-pull-request +- name: go-elasticsearch-alerts-pr type: pull-request source: repository: morningconsult/go-elasticsearch-alerts access_token: ((github-access-token)) -- name: new-release +- name: go-elasticsearch-alerts + type: git + source: + private_key: ((github-private-key)) + access_token: ((github-access-token)) + uri: ((github-repo)) + branch: master + clean_tags: true + +- name: go-elasticsearch-alerts-bumped type: git source: private_key: ((github-private-key)) access_token: ((github-access-token)) uri: ((github-repo)) branch: master - tag_filter: 'v*' - name: golang type: registry-image source: repository: golang - tag: 1.11.4-alpine3.8 + tag: 1.12-alpine3.9 + +- name: python + type: registry-image + source: + repository: python + tag: 3.7.3-alpine3.9 - name: slack type: slack-alert @@ -55,50 +69,98 @@ resources: # # ==================================== jobs: -- name: test-pr +- name: validate-pull-request build_logs_to_retain: 30 serial: true plan: - in_parallel: - - get: test-pull-request + - get: go-elasticsearch-alerts-pr trigger: true - get: golang - - put: test-pull-request + - put: go-elasticsearch-alerts-pr params: - path: test-pull-request + path: go-elasticsearch-alerts-pr status: pending - context: test-pr - - task: test-pr + - task: validate-pull-request image: golang - file: test-pull-request/ci/tasks/test-pull-request.yml + file: go-elasticsearch-alerts-pr/ci/tasks/validate.yml + input_mapping: {repo: go-elasticsearch-alerts-pr} on_failure: - put: test-pull-request + put: go-elasticsearch-alerts-pr params: - path: test-pull-request + path: go-elasticsearch-alerts-pr status: failure - context: test-pr on_abort: - put: test-pull-request + put: go-elasticsearch-alerts-pr params: - path: test-pull-request + path: go-elasticsearch-alerts-pr status: error - context: test-pr - - put: test-pull-request + - put: go-elasticsearch-alerts-pr params: - path: test-pull-request + path: go-elasticsearch-alerts-pr status: success - context: test-pr -- name: build-release +- name: validate-master + build_logs_to_retain: 30 + serial: true + plan: + - in_parallel: + - get: go-elasticsearch-alerts + trigger: true + - get: golang + - task: validate-master + image: golang + file: go-elasticsearch-alerts/ci/tasks/validate.yml + input_mapping: {repo: go-elasticsearch-alerts} + on_failure: + put: slack + params: {alert_type: failed} + on_abort: + put: slack + params: {alert_type: aborted} + +- name: build-new-release + build_logs_to_retain: 30 + serial: true + plan: + - in_parallel: + - get: go-elasticsearch-alerts + trigger: true + passed: [validate-master] + - get: python + - task: build-release + image: python + file: go-elasticsearch-alerts/ci/tasks/build-release.yml + input_mapping: {repo: go-elasticsearch-alerts} + output_mapping: {repo-dirty: go-elasticsearch-alerts-dirty} + params: + GITHUB_ACTOR: ((github-actor)) + GITHUB_EMAIL: ((github-email)) + on_success: + put: go-elasticsearch-alerts-bumped + params: + repository: go-elasticsearch-alerts-dirty + merge: true + on_failure: + put: slack + params: {alert_type: failed} + on_abort: + put: slack + params: {alert_type: aborted} + +- name: publish-release + build_logs_to_retain: 30 serial: true plan: - in_parallel: - - get: new-release + - get: go-elasticsearch-alerts-bumped trigger: true + passed: [build-new-release] - get: golang - task: test-and-build image: golang - file: new-release/ci/tasks/build-release.yml + file: go-elasticsearch-alerts-bumped/ci/tasks/publish-release.yml + input_mapping: {repo: go-elasticsearch-alerts-bumped} params: GITHUB_TOKEN: ((github-access-token)) on_failure: diff --git a/ci/release/build-release.sh b/ci/release/build-release.sh new file mode 100755 index 0000000..3ca2396 --- /dev/null +++ b/ci/release/build-release.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Copyright 2019 The Morning Consult, LLC or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may +# not use this file except in compliance with the License. A copy of the +# License is located at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +set -eu + +apk add -qU --no-cache --no-progress git + +git config --global user.email "${GITHUB_EMAIL}" +git config --global user.name "${GITHUB_ACTOR}" + +readonly GIT_CHGLOG_VERSION="0.8.0" + +# Download git-chglog +wget --output-document=/usr/local/bin/git-chglog --quiet \ + "https://github.com/git-chglog/git-chglog/releases/download/${GIT_CHGLOG_VERSION}/git-chglog_linux_amd64" + +# Make it executable +chmod a+x /usr/local/bin/git-chglog + +# Clone the repo +git clone repo repo-dirty + +cd repo-dirty + +# Install dependencies +pip install --quiet --requirement ./ci/release/requirements.txt + +# Get the latest version +readonly VERSION="$( python ./ci/release/version.py )" + +if [ "${VERSION}" = "" ]; then + exit 0 +fi + +echo "${VERSION}" > ./VERSION + +git-chglog --output ./CHANGELOG.md + +# Add and commit changed files +git add ./VERSION ./CHANGELOG.md +git commit -m "chore: Bump version and update changelog [ci skip]" + +# Tag the repo with the latest version +git tag "${VERSION}" diff --git a/ci/build-release.sh b/ci/release/publish-release.sh similarity index 73% rename from ci/build-release.sh rename to ci/release/publish-release.sh index 5d466dd..465c53e 100755 --- a/ci/build-release.sh +++ b/ci/release/publish-release.sh @@ -15,26 +15,22 @@ set -e readonly PROJECT="github.com/morningconsult/go-elasticsearch-alerts" -readonly GORELEASER_VERSION=v0.104.0 +readonly GORELEASER_VERSION="v0.108.0" echo "==> Installing APK dependencies" apk add -qU --no-cache --no-progress \ - make \ - bash \ git \ - gnupg \ - bzr + gnupg echo "==> Installing goreleaser $GORELEASER_VERSION" -wget --quiet -O /tmp/goreleaser.tar.gz https://github.com/goreleaser/goreleaser/releases/download/${GORELEASER_VERSION}/goreleaser_Linux_x86_64.tar.gz -tar xzf /tmp/goreleaser.tar.gz -C /tmp -mv /tmp/goreleaser /usr/local/bin +wget --quiet -O /tmp/goreleaser.tar.gz "https://github.com/goreleaser/goreleaser/releases/download/${GORELEASER_VERSION}/goreleaser_Linux_x86_64.tar.gz" +tar xzf /tmp/goreleaser.tar.gz -C /usr/local/bin echo "==> Running unit tests" -CGO_ENABLED=0 make test +CGO_ENABLED=0 GO111MODULE=on go test ./... goreleaser release \ --rm-dist diff --git a/ci/release/requirements.txt b/ci/release/requirements.txt new file mode 100644 index 0000000..dc64312 --- /dev/null +++ b/ci/release/requirements.txt @@ -0,0 +1,2 @@ +GitPython +semantic_version diff --git a/ci/release/version.py b/ci/release/version.py new file mode 100644 index 0000000..c7697df --- /dev/null +++ b/ci/release/version.py @@ -0,0 +1,56 @@ +from semantic_version import Version +from git import Repo +import os +import re + +def enumerate_semver(version): + return (int(i.strip('vV')) for i in version.split('.')) + +def get_latest_version(repo): + semver_re = re.compile("^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*)?(\+[0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*)?$") + + semvers = [str(tag) for tag in repo.tags if semver_re.search(str(tag))] + + semvers.sort( + reverse=True, + key=lambda x: tuple(enumerate_semver(x)) + ) + + if len(semvers) < 1: + return '' + + return semvers[0] + +def next_version(version, repo, branch='master'): + latest_tagged_commit = repo.commit(str(version)) + + pre_v1 = version.major < 1 + + new_feature = False + for commit in repo.iter_commits(branch): + if 'BREAKING CHANGE' in commit.summary: + if pre_v1: + version = version.next_minor() + else: + version = version.next_major() + return version + if commit.summary.find('feat') == 0: + new_feature = True + if latest_tagged_commit == commit: + break + if new_feature and not pre_v1: + return version.next_minor() + return version.next_patch() + +def main(): + repo = Repo() + latest_tag = get_latest_version(repo) + if latest_tag == '': + print('No semantic version tags found') + exit(1) + v = Version(latest_tag) + version = next_version(v, repo) + print(version) + +if __name__ == '__main__': main() + diff --git a/ci/run-tests.sh b/ci/run-tests.sh deleted file mode 100755 index 61d233f..0000000 --- a/ci/run-tests.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# Copyright 2019 The Morning Consult, LLC or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may -# not use this file except in compliance with the License. A copy of the -# License is located at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -set -e - -readonly PROJECT="github.com/morningconsult/go-elasticsearch-alerts" - -# NOTE: This script is intended to be run in a -# golang:1.11.3-alpine3.8 Docker container - -# Install dependencies -apk add -qU --no-cache --no-progress \ - make \ - git \ - bzr - -# Run tests -CGO_ENABLED=0 make test diff --git a/ci/tasks/build-release.yml b/ci/tasks/build-release.yml index 9738624..e894757 100644 --- a/ci/tasks/build-release.yml +++ b/ci/tasks/build-release.yml @@ -1,9 +1,11 @@ --- platform: linux inputs: - - name: new-release +- name: repo params: - GITHUB_TOKEN: + GITHUB_ACTOR: + GITHUB_EMAIL: run: - path: ci/build-release.sh - dir: new-release \ No newline at end of file + path: repo/ci/release/build-release.sh +outputs: +- name: repo-dirty diff --git a/ci/tasks/publish-release.yml b/ci/tasks/publish-release.yml new file mode 100644 index 0000000..8d72f0d --- /dev/null +++ b/ci/tasks/publish-release.yml @@ -0,0 +1,9 @@ +--- +platform: linux +inputs: + - name: repo +params: + GITHUB_TOKEN: +run: + path: ci/release/publish-release.sh + dir: repo diff --git a/ci/tasks/test-pull-request.yml b/ci/tasks/validate.yml similarity index 51% rename from ci/tasks/test-pull-request.yml rename to ci/tasks/validate.yml index e6655df..2589ed7 100644 --- a/ci/tasks/test-pull-request.yml +++ b/ci/tasks/validate.yml @@ -1,7 +1,7 @@ --- platform: linux inputs: - - name: test-pull-request + - name: repo run: path: ci/run-tests.sh - dir: test-pull-request + dir: repo