Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master' into feature/windows-2008
Browse files Browse the repository at this point in the history
* upstream/master: (417 commits)
  libbeat/cmd/instance: report cgroup stats (elastic#21113)
  Configurable index template loading (elastic#21212)
  [Ingest Manager] Thread safe sorted set (elastic#21290)
  Change mirror of kafka download (elastic#19645)
  [Ingest manager] Copy Action store on upgrade (elastic#21298)
  [CI] Pipeline 2.0 for monorepos (elastic#20104)
  Stop running agent container as root by default (elastic#21213)
  Stop running auditbeat container as root by default (elastic#21202)
  Fix autodiscover flaky tests (elastic#21242)
  [Ingest Manager] Enabled dev builds (elastic#21241)
  Fix librpm installation in auditbeat build (elastic#21239)
  Fix prometheus default config (elastic#21253)
  Fix dev guide test command (elastic#21254)
  Move aws lambda metricset to GA (elastic#21255)
  [Docs] Typo in table syntax (elastic#20227)
  [ECS] Adds related.hosts to capture all hostnames and host identifiers on an event. (elastic#21160)
  Add recursive split to httpjson (elastic#21214)
  [DOCS] Add beat specific start widgets (elastic#21217)
  Fix timestamp handling in remote_write (elastic#21166)
  Fix aws, azure and googlecloud compute dashboards (elastic#21098)
  ...
  • Loading branch information
v1v committed Sep 28, 2020
2 parents 2a99271 + b4c7a93 commit 2cbe6d9
Show file tree
Hide file tree
Showing 2,198 changed files with 769,215 additions and 41,550 deletions.
2 changes: 1 addition & 1 deletion .backportrc.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"upstream": "elastic/beats",
"branches": [{ "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
"branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
"labels": ["backport"],
"autoAssign": true,
"prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"
Expand Down
108 changes: 57 additions & 51 deletions .ci/apm-beats-update.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
@Library('apm@current') _

pipeline {
agent none
agent { label 'master' }
environment {
REPO = 'apm-server'
BASE_DIR = "src/github.com/elastic/${env.REPO}"
Expand All @@ -27,69 +27,75 @@ pipeline {
}
triggers {
issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
upstream("Beats/beats-beats-mbp/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
upstream("Beats/beats/${ env.JOB_BASE_NAME.startsWith('PR-') ? 'none' : env.JOB_BASE_NAME }")
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
agent { label 'ubuntu-18 && immutable' }
when {
beforeAgent true
expression {
def ret = isCommentTrigger() || isUserTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, and GitHub comment")
anyOf {
triggeredBy cause: "IssueCommentCause"
expression {
def ret = isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
}
return ret
}
return ret
}
}
/**
Checkout the code and stash it, to use it on other stages.
*/
stage('Checkout') {
steps {
deleteDir()
gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
script {
dir("${BEATS_DIR}"){
env.GO_VERSION = readFile(".go-version").trim()
def regexps =[
"^devtools/mage.*",
"^libbeat/scripts/Makefile",
]
env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
// Skip all the stages except docs for PR's with asciidoc changes only
env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
stages {
/**
Checkout the code and stash it, to use it on other stages.
*/
stage('Checkout') {
options { skipDefaultCheckout() }
steps {
deleteDir()
gitCheckout(basedir: "${BEATS_DIR}", githubNotifyFirstTimeContributor: false)
script {
dir("${BEATS_DIR}"){
env.GO_VERSION = readFile(".go-version").trim()
def regexps =[
"^devtools/mage.*",
"^libbeat/scripts/Makefile",
]
env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
// Skip all the stages except docs for PR's with asciidoc changes only
env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
}
}
}
}
}
/**
updates beats updates the framework part and go parts of beats.
Then build and test.
Finally archive the results.
*/
stage('Update Beats') {
options { skipDefaultCheckout() }
when {
beforeAgent true
anyOf {
branch 'master'
branch "\\d+\\.\\d+"
branch "v\\d?"
tag "v\\d+\\.\\d+\\.\\d+*"
allOf {
expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
changeRequest()
}
/**
updates beats updates the framework part and go parts of beats.
Then build and test.
Finally archive the results.
*/
stage('Update Beats') {
options { skipDefaultCheckout() }
when {
beforeAgent true
anyOf {
branch 'master'
branch "\\d+\\.\\d+"
branch "v\\d?"
tag "v\\d+\\.\\d+\\.\\d+*"
allOf {
expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
changeRequest()
}

}
}
}
steps {
withGithubNotify(context: 'Check Apm Server Beats Update') {
beatsUpdate()
steps {
withGithubNotify(context: 'Check Apm Server Beats Update') {
beatsUpdate()
}
}
}
}
Expand Down
106 changes: 106 additions & 0 deletions .ci/beats-tester.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
#!/usr/bin/env groovy

@Library('apm@current') _

pipeline {
agent none
environment {
BASE_DIR = 'src/github.com/elastic/beats'
PIPELINE_LOG_LEVEL = "INFO"
BEATS_TESTER_JOB = 'Beats/beats-tester-mbp/master'
}
options {
timeout(time: 1, unit: 'HOURS')
buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
timestamps()
ansiColor('xterm')
disableResume()
durabilityHint('PERFORMANCE_OPTIMIZED')
disableConcurrentBuilds()
}
triggers {
issueCommentTrigger('(?i)^\\/beats-tester$')
upstream("Beats/packaging/${env.JOB_BASE_NAME}")
}
stages {
stage('Filter build') {
agent { label 'ubuntu && immutable' }
when {
beforeAgent true
anyOf {
triggeredBy cause: "IssueCommentCause"
expression {
def ret = isUserTrigger() || isUpstreamTrigger()
if(!ret){
currentBuild.result = 'NOT_BUILT'
currentBuild.description = "The build has been skipped"
currentBuild.displayName = "#${BUILD_NUMBER}-(Skipped)"
echo("the build has been skipped due the trigger is a branch scan and the allow ones are manual, GitHub comment, and upstream job")
}
return ret
}
}
}
stages {
stage('Checkout') {
options { skipDefaultCheckout() }
steps {
deleteDir()
gitCheckout(basedir: "${BASE_DIR}")
setEnvVar('VERSION', sh(script: "grep ':stack-version:' ${BASE_DIR}/libbeat/docs/version.asciidoc | cut -d' ' -f2", returnStdout: true).trim())
}
}
stage('Build master') {
options { skipDefaultCheckout() }
when { branch 'master' }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
stage('Build *.x branch') {
options { skipDefaultCheckout() }
when { branch '*.x' }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
stage('Build PullRequest') {
options { skipDefaultCheckout() }
when { changeRequest() }
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT",
apm: "https://storage.googleapis.com/apm-ci-artifacts/jobs/pull-requests/pr-${env.CHANGE_ID}",
beats: "https://storage.googleapis.com/beats-ci-artifacts/pull-requests/pr-${env.CHANGE_ID}")
}
}
stage('Build release branch') {
options { skipDefaultCheckout() }
when {
not {
allOf {
branch comparator: 'REGEXP', pattern: '(master|.*x)'
changeRequest()
}
}
}
steps {
runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
}
}
}
}
}
}

def runBeatsTesterJob(Map args = [:]) {
if (args.apm && args.beats) {
build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false,
parameters: [
string(name: 'APM_URL_BASE', value: args.apm),
string(name: 'BEATS_URL_BASE', value: args.beats),
string(name: 'VERSION', value: args.version)
])
} else {
build(job: env.BEATS_TESTER_JOB, propagate: false, wait: false, parameters: [ string(name: 'VERSION', value: args.version) ])
}
}
2 changes: 1 addition & 1 deletion .ci/jobs/apm-beats-update.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
notification-context: 'apm-beats-update'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
56 changes: 56 additions & 0 deletions .ci/jobs/beats-tester.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
---
- job:
name: Beats/beats-tester
display-name: Beats Tester
description: Run the beats-tester
view: Beats
disabled: false
project-type: multibranch
script-path: .ci/beats-tester.groovy
scm:
- github:
branch-discovery: 'no-pr'
discover-pr-forks-strategy: 'merge-current'
discover-pr-forks-trust: 'permission'
discover-pr-origin: 'merge-current'
discover-tags: true
head-filter-regex: '(master|7\.([x9]|1\d+)|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
disable-pr-notifications: true
notification-context: 'beats-tester'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
- skip-initial-build: true
- tags:
ignore-tags-older-than: -1
ignore-tags-newer-than: 30
- named-branches:
- exact-name:
name: 'master'
case-sensitive: true
- regex-name:
regex: '7\.([x9]|1\d+)'
case-sensitive: true
- regex-name:
regex: '8\.\d+'
case-sensitive: true
- change-request:
ignore-target-only-changes: true
clean:
after: true
before: true
prune: true
shallow-clone: true
depth: 3
do-not-fetch-tags: true
submodule:
disable: false
recursive: true
parent-credentials: true
timeout: 100
timeout: '15'
use-author: true
wipe-workspace: true
4 changes: 2 additions & 2 deletions .ci/jobs/beats-windows-mbp.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@
discover-tags: false
# Run MBP for the master branch and PRs
head-filter-regex: '(master|PR-.*)'
notification-context: 'beats-ci'
notification-context: 'beats-ci/windows'
repo: beats
repo-owner: elastic
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
4 changes: 2 additions & 2 deletions .ci/jobs/beats.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
---
- job:
name: Beats/beats
display-name: 'Beats (replacement)'
display-name: 'Beats'
description: 'Beats Main Pipeline'
view: Beats
concurrent: true
Expand All @@ -22,7 +22,7 @@
notification-context: "beats-ci"
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
2 changes: 1 addition & 1 deletion .ci/jobs/packaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
notification-context: 'beats-packaging'
repo: 'beats'
repo-owner: 'elastic'
credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
credentials-id: github-app-beats-ci
ssh-checkout:
credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
build-strategies:
Expand Down
Loading

0 comments on commit 2cbe6d9

Please sign in to comment.