From 44fbaaf9f11e038ad5efe93a128764e3198e6f84 Mon Sep 17 00:00:00 2001 From: Alex Dulin Date: Sat, 13 Apr 2019 16:41:41 -0400 Subject: [PATCH 1/4] templates: Reduce number of shards to 1 --- command/query/job.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/command/query/job.go b/command/query/job.go index fe2ef0d..fdea1f9 100644 --- a/command/query/job.go +++ b/command/query/job.go @@ -38,7 +38,7 @@ import ( ) const ( - templateVersion string = "0.0.1" + templateVersion string = "0.0.2" defaultStateIndexAlias string = "go-es-alerts" defaultTimestampFormat string = time.RFC3339 defaultBodyField string = "hits.hits._source" @@ -272,7 +272,7 @@ func (q *QueryHandler) Run(ctx context.Context, outputCh chan *alert.Alert, wg * // will be named 'go-es-alerts-status-{date}'; therefore, this template // enables searching all state indices via this alias func (q *QueryHandler) PutTemplate(ctx context.Context) error { - payload := fmt.Sprintf(`{"index_patterns":["%s-status-%s-*"],"order":0,"aliases":{%q:{}},"settings":{"index":{"number_of_shards":5,"number_of_replicas":1,"auto_expand_replicas":"0-2","translog":{"flush_threshold_size":"752mb"},"sort":{"field":["next_query","rule_name","hostname"],"order":["desc","desc","desc"]}}},"mappings":{"_doc":{"dynamic_templates":[{"strings_as_keywords":{"match_mapping_type":"string","mapping":{"type":"keyword"}}}],"properties":{"@timestamp":{"type":"date"},"rule_name":{"type":"keyword"},"next_query":{"type":"date"},"hostname":{"type":"keyword"},"hits_count":{"type":"long","null_value":0},"hits":{"enabled":false}}}}}`, defaultStateIndexAlias, templateVersion, q.TemplateName()) + payload := fmt.Sprintf(`{"index_patterns":["%s-status-%s-*"],"order":0,"aliases":{%q:{}},"settings":{"index":{"number_of_shards":1,"number_of_replicas":1,"auto_expand_replicas":"0-3","codec":"best_compression","translog":{"flush_threshold_size":"752mb"},"sort":{"field":["next_query","rule_name","hostname"],"order":["desc","desc","desc"]}}},"mappings":{"_doc":{"dynamic_templates":[{"strings_as_keywords":{"match_mapping_type":"string","mapping":{"type":"keyword"}}}],"properties":{"@timestamp":{"type":"date"},"rule_name":{"type":"keyword"},"next_query":{"type":"date"},"hostname":{"type":"keyword"},"hits_count":{"type":"long","null_value":0},"hits":{"enabled":false}}}}}`, defaultStateIndexAlias, templateVersion, q.TemplateName()) resp, err := q.makeRequest(ctx, "PUT", fmt.Sprintf("%s/_template/%s", q.esURL, q.TemplateName()), []byte(payload)) if err != nil { From d423f9c29fa4f0728005047e14ca0424cec9edbd Mon Sep 17 00:00:00 2001 From: Alex Dulin Date: Sat, 13 Apr 2019 16:52:44 -0400 Subject: [PATCH 2/4] ci: Clean up some pipeline stuff --- Makefile | 16 +++++++++------- ci/build-release.sh | 2 +- ci/pipeline.yml | 10 +++------- ci/pr_test_failure | 2 -- ci/pr_test_success | 2 -- ci/run-tests.sh | 7 +++++-- 6 files changed, 18 insertions(+), 21 deletions(-) delete mode 100644 ci/pr_test_failure delete mode 100644 ci/pr_test_success diff --git a/Makefile b/Makefile index 03d54bc..6be746c 100644 --- a/Makefile +++ b/Makefile @@ -59,19 +59,21 @@ changelog: git_chglog_check CONCOURSE_PIPELINE := go-elasticsearch-alerts + check_fly: - if [ -z "$(FLY)" ]; then \ - sudo mkdir -p /usr/local/bin; \ - sudo wget -q -O /usr/local/bin/fly "https://ci.morningconsultintelligence.com/api/v1/cli?arch=amd64&platform=linux"; \ - sudo chmod +x /usr/local/bin/fly; \ - /usr/local/bin/fly --version; \ - fi +ifeq ($(FLY),) + sudo mkdir -p /usr/local/bin + sudo wget -q -O /usr/local/bin/fly "https://ci.morningconsultintelligence.com/api/v1/cli?arch=amd64&platform=linux"; \ + sudo chmod +x /usr/local/bin/fly + /usr/local/bin/fly --version +endif .PHONY: check_fly set_pipeline: check_fly $(FLY) --target mci-ci-oss validate-pipeline \ - --config ci/pipeline.yml + --config ci/pipeline.yml \ + --strict $(FLY) --target mci-ci-oss set-pipeline \ --config ci/pipeline.yml \ diff --git a/ci/build-release.sh b/ci/build-release.sh index 2f2a085..5d466dd 100755 --- a/ci/build-release.sh +++ b/ci/build-release.sh @@ -19,7 +19,7 @@ readonly GORELEASER_VERSION=v0.104.0 echo "==> Installing APK dependencies" -apk add -qU --no-progress \ +apk add -qU --no-cache --no-progress \ make \ bash \ git \ diff --git a/ci/pipeline.yml b/ci/pipeline.yml index b5082ed..73737e8 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,13 +6,12 @@ # ==================================== resource_types: - name: slack-alert - type: docker-image + type: registry-image source: repository: arbourd/concourse-slack-alert-resource - channel: '#build' - name: pull-request - type: docker-image + type: registry-image source: repository: teliaoss/github-pr-resource @@ -39,7 +38,7 @@ resources: tag_filter: 'v*' - name: golang - type: docker-image + type: registry-image source: repository: golang tag: 1.11.4-alpine3.8 @@ -77,21 +76,18 @@ jobs: path: test-pull-request status: success context: $BUILD_JOB_NAME - comment: test-pull-request/ci/pr_test_success on_failure: put: test-pull-request params: path: test-pull-request status: failure context: $BUILD_JOB_NAME - comment: test-pull-request/ci/pr_test_failure on_abort: put: test-pull-request params: path: test-pull-request status: error context: $BUILD_JOB_NAME - comment: test-pull-request/ci/pr_test_failure - name: build-release serial: true diff --git a/ci/pr_test_failure b/ci/pr_test_failure deleted file mode 100644 index e4cbf6a..0000000 --- a/ci/pr_test_failure +++ /dev/null @@ -1,2 +0,0 @@ -### Concourse CI -Tests failing :x: \ No newline at end of file diff --git a/ci/pr_test_success b/ci/pr_test_success deleted file mode 100644 index 5f03a0f..0000000 --- a/ci/pr_test_success +++ /dev/null @@ -1,2 +0,0 @@ -### Concourse CI -All tests passing :white_check_mark: \ No newline at end of file diff --git a/ci/run-tests.sh b/ci/run-tests.sh index d20e4cf..61d233f 100755 --- a/ci/run-tests.sh +++ b/ci/run-tests.sh @@ -14,13 +14,16 @@ set -e -PROJECT="github.com/morningconsult/go-elasticsearch-alerts" +readonly PROJECT="github.com/morningconsult/go-elasticsearch-alerts" # NOTE: This script is intended to be run in a # golang:1.11.3-alpine3.8 Docker container # Install dependencies -apk add -qU --no-cache make git bzr +apk add -qU --no-cache --no-progress \ + make \ + git \ + bzr # Run tests CGO_ENABLED=0 make test From f36f28240a07460872acc78c31f5c4154e1714c2 Mon Sep 17 00:00:00 2001 From: Alex Dulin Date: Sat, 13 Apr 2019 17:00:24 -0400 Subject: [PATCH 3/4] make: Missed a line break in fly check --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6be746c..1c21b2f 100644 --- a/Makefile +++ b/Makefile @@ -63,7 +63,7 @@ CONCOURSE_PIPELINE := go-elasticsearch-alerts check_fly: ifeq ($(FLY),) sudo mkdir -p /usr/local/bin - sudo wget -q -O /usr/local/bin/fly "https://ci.morningconsultintelligence.com/api/v1/cli?arch=amd64&platform=linux"; \ + sudo wget -q -O /usr/local/bin/fly "https://ci.morningconsultintelligence.com/api/v1/cli?arch=amd64&platform=linux"; sudo chmod +x /usr/local/bin/fly /usr/local/bin/fly --version endif From 7765518af0b8726beb5d547e191a0caf4f2817ec Mon Sep 17 00:00:00 2001 From: Alex Dulin Date: Sat, 13 Apr 2019 17:36:59 -0400 Subject: [PATCH 4/4] ci: FIx PR context not being interpolated --- ci/pipeline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 73737e8..12b3e69 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -75,19 +75,19 @@ jobs: params: path: test-pull-request status: success - context: $BUILD_JOB_NAME + context: test-pr on_failure: put: test-pull-request params: path: test-pull-request status: failure - context: $BUILD_JOB_NAME + context: test-pr on_abort: put: test-pull-request params: path: test-pull-request status: error - context: $BUILD_JOB_NAME + context: test-pr - name: build-release serial: true