diff --git a/.backportrc.json b/.backportrc.json index 3dce189d671..8bfd891160c 100644 --- a/.backportrc.json +++ b/.backportrc.json @@ -1,6 +1,6 @@ { "upstream": "elastic/beats", - "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}], + "branches": [ { "name": "7.x", "checked": true }, "7.10", "7.9", "7.8", "7.7"], "labels": ["backport"], "autoAssign": true, "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}" diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml index 8bdc322f65a..678a9fbcd67 100644 --- a/.ci/jobs/apm-beats-update.yml +++ b/.ci/jobs/apm-beats-update.yml @@ -18,7 +18,7 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true - head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' + head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' disable-pr-notifications: true notification-context: 'apm-beats-update' repo: 'beats' @@ -38,6 +38,9 @@ - regex-name: regex: '7\.[x789]' case-sensitive: true + - regex-name: + regex: '7\.1\d' + case-sensitive: true - regex-name: regex: '8\.\d+' case-sensitive: true @@ -48,7 +51,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats-tester.yml b/.ci/jobs/beats-tester.yml index 522abfa5e5c..808123a225e 100644 --- a/.ci/jobs/beats-tester.yml +++ b/.ci/jobs/beats-tester.yml @@ -44,7 +44,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/beats-windows-mbp.yml b/.ci/jobs/beats-windows-mbp.yml deleted file mode 100644 index 64efa009979..00000000000 --- a/.ci/jobs/beats-windows-mbp.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -- job: - name: Beats/beats-windows-mbp - display-name: Beats Pipeline for Windows - description: Jenkins pipeline for the Beats project running on windows agents. - view: Beats - project-type: multibranch - script-path: .ci/windows.groovy - scm: - - github: - branch-discovery: no-pr - discover-pr-forks-strategy: merge-current - discover-pr-forks-trust: permission - discover-pr-origin: merge-current - discover-tags: false - # Run MBP for the master branch and PRs - head-filter-regex: '(master|PR-.*)' - notification-context: 'beats-ci/windows' - repo: beats - repo-owner: elastic - credentials-id: github-app-beats-ci - ssh-checkout: - credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba - build-strategies: - - tags: - ignore-tags-older-than: -1 - ignore-tags-newer-than: -1 - - regular-branches: true - - change-request: - ignore-target-only-changes: true - property-strategies: - # Builds for PRs won't be triggered automatically. - # Only the master branch will be triggered automatically. - named-branches: - defaults: - - suppress-scm-triggering: true - exceptions: - - exception: - branch-name: master - properties: - - suppress-scm-triggering: false - clean: - after: true - before: true - prune: true - shallow-clone: true - depth: 4 - do-not-fetch-tags: true - submodule: - disable: false - recursive: true - parent-credentials: true - timeout: 100 - timeout: '15' - use-author: true - wipe-workspace: 'True' diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml index b075d8bbdf2..27095b2fecb 100644 --- a/.ci/jobs/beats.yml +++ b/.ci/jobs/beats.yml @@ -17,7 +17,7 @@ discover-pr-forks-strategy: 'merge-current' discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' - head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' + head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' discover-tags: true notification-context: "beats-ci" repo: 'beats' @@ -38,6 +38,9 @@ - regex-name: regex: '7\.[x789]' case-sensitive: true + - regex-name: + regex: '7\.1\d' + case-sensitive: true - regex-name: regex: '8\.\d+' case-sensitive: true @@ -46,7 +49,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml index 46303790610..45175d169f6 100644 --- a/.ci/jobs/golang-crossbuild-mbp.yml +++ b/.ci/jobs/golang-crossbuild-mbp.yml @@ -31,7 +31,7 @@ before: true prune: true shallow-clone: true - depth: 4 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml index 0dce4d4672b..baa3ce45035 100644 --- a/.ci/jobs/packaging.yml +++ b/.ci/jobs/packaging.yml @@ -14,7 +14,7 @@ discover-pr-forks-trust: 'permission' discover-pr-origin: 'merge-current' discover-tags: true - head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' + head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)' disable-pr-notifications: true notification-context: 'beats-packaging' repo: 'beats' @@ -34,6 +34,9 @@ - regex-name: regex: '7\.[x789]' case-sensitive: true + - regex-name: + regex: '7\.1\d' + case-sensitive: true - regex-name: regex: '8\.\d+' case-sensitive: true @@ -44,7 +47,7 @@ before: true prune: true shallow-clone: true - depth: 3 + depth: 10 do-not-fetch-tags: true submodule: disable: false diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 2be78aac68f..37eeaa7d223 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -5,12 +5,14 @@ pipeline { agent none environment { - BASE_DIR = 'src/github.com/elastic/beats' + REPO = 'beats' + BASE_DIR = "src/github.com/elastic/${env.REPO}" JOB_GCS_BUCKET = 'beats-ci-artifacts' JOB_GCS_BUCKET_STASH = 'beats-ci-temp' JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' DOCKER_REGISTRY = 'docker.elastic.co' + GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests' SNAPSHOT = "true" PIPELINE_LOG_LEVEL = "INFO" } @@ -119,6 +121,7 @@ pipeline { release() pushCIDockerImages() } + runE2ETestForPackages() } } stage('Package Mac OS'){ @@ -195,29 +198,97 @@ def tagAndPush(name){ tagName = "pr-${env.CHANGE_ID}" } - def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}" - def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}" - def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}" dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - retry(3){ - sh(label:'Change tag and push', script: """ - docker tag ${oldName} ${newName} - docker push ${newName} - docker tag ${oldName} ${commitName} - docker push ${commitName} - """) + + // supported image flavours + def variants = ["", "-oss", "-ubi8"] + variants.each { variant -> + def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}" + def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}" + def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}" + + def iterations = 0 + retryWithSleep(retries: 3, seconds: 5, backoff: true) + iterations++ + def status = sh(label:'Change tag and push', script: """ + docker tag ${oldName} ${newName} + docker push ${newName} + docker tag ${oldName} ${commitName} + docker push ${commitName} + """, returnStatus: true) + + if ( status > 0 && iterations < 3) { + error('tag and push failed, retry') + } else if ( status > 0 ) { + log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621") + } + } +} + +def runE2ETestForPackages(){ + def suite = '' + + catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') { + if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") { + suite = 'helm,fleet' + } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") { + suite = '' + } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") { + suite = 'fleet' + } else { + echo("Skipping E2E tests for ${env.BEATS_FOLDER}.") + return + } + + triggerE2ETests(suite) } } def release(){ withBeatsEnv(){ - dir("${env.BEATS_FOLDER}") { - sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package') + withEnv([ + "DEV=true" + ]) { + dir("${env.BEATS_FOLDER}") { + sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package') + } } publishPackages("${env.BEATS_FOLDER}") } } +def triggerE2ETests(String suite) { + echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.") + + def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}" + def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}" + + def parameters = [ + booleanParam(name: 'forceSkipGitChecks', value: true), + booleanParam(name: 'forceSkipPresubmit', value: true), + booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()), + string(name: 'runTestsSuites', value: suite), + string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME), + string(name: 'GITHUB_CHECK_REPO', value: env.REPO), + string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT), + ] + if (isPR()) { + def version = "pr-${env.CHANGE_ID}" + parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true)) + parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}")) + parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}")) + } + + build(job: "${e2eTestsPipeline}", + parameters: parameters, + propagate: false, + wait: false + ) + + def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}" + githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}") +} + def withMacOSEnv(Closure body){ withEnvMask( vars: [ [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password], diff --git a/.ci/windows.groovy b/.ci/windows.groovy deleted file mode 100644 index 92826e80926..00000000000 --- a/.ci/windows.groovy +++ /dev/null @@ -1,874 +0,0 @@ -#!/usr/bin/env groovy - -@Library('apm@current') _ - -import groovy.transform.Field - -/** - This is required to store the stashed id with the test results to be digested with runbld -*/ -@Field def stashedTestReports = [:] - -/** - List of supported windows versions to be tested with - NOTE: - - 'windows-10' is too slow - - 'windows-2012-r2', 'windows-2008-r2', 'windows-7', 'windows-7-32-bit' are disabled - since we are working on releasing each windows version incrementally. -*/ -@Field def windowsVersions = ['windows-2019', 'windows-2016', 'windows-2012-r2'] - -pipeline { - agent { label 'ubuntu && immutable' } - environment { - BASE_DIR = 'src/github.com/elastic/beats' - GOX_FLAGS = "-arch amd64" - DOCKER_COMPOSE_VERSION = "1.21.0" - TERRAFORM_VERSION = "0.12.24" - PIPELINE_LOG_LEVEL = "INFO" - DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod' - DOCKER_REGISTRY = 'docker.elastic.co' - AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth' - RUNBLD_DISABLE_NOTIFICATIONS = 'true' - JOB_GCS_BUCKET = 'beats-ci-temp' - JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin' - } - options { - timeout(time: 2, unit: 'HOURS') - buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) - timestamps() - ansiColor('xterm') - disableResume() - durabilityHint('PERFORMANCE_OPTIMIZED') - quietPeriod(10) - rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true]) - } - triggers { - issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?tests(?:\\W+please)?.*') - } - parameters { - booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.') - booleanParam(name: 'windowsTest', defaultValue: true, description: 'Allow Windows stages.') - booleanParam(name: 'macosTest', defaultValue: true, description: 'Allow macOS stages.') - - booleanParam(name: 'allCloudTests', defaultValue: false, description: 'Run all cloud integration tests.') - booleanParam(name: 'awsCloudTests', defaultValue: false, description: 'Run AWS cloud integration tests.') - string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.') - - booleanParam(name: 'debug', defaultValue: false, description: 'Allow debug logging for Jenkins steps') - booleanParam(name: 'dry_run', defaultValue: false, description: 'Skip build steps, it is for testing pipeline flow') - } - stages { - /** - Checkout the code and stash it, to use it on other stages. - */ - stage('Checkout') { - options { skipDefaultCheckout() } - steps { - pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ]) - deleteDir() - gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true) - stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${BASE_DIR}"){ - loadConfigEnvVars() - } - whenTrue(params.debug){ - dumpFilteredEnvironment() - } - } - } - stage('Lint'){ - options { skipDefaultCheckout() } - steps { - // NOTE: commented to run the windows pipeline a bit faster. - // when required then it can be enabled. - // makeTarget("Lint", "check") - echo 'SKIPPED' - } - } - stage('Build and Test Windows'){ - failFast false - parallel { - stage('Elastic Agent x-pack Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_ELASTIC_AGENT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Elastic Agent x-pack Windows Unit test", "x-pack/elastic-agent", "build unitTest") - } - } - stage('Filebeat Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_FILEBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Filebeat oss Windows Unit test", "filebeat", "build unitTest") - } - } - stage('Filebeat x-pack Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_FILEBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Filebeat x-pack Windows", "x-pack/filebeat", "build unitTest") - } - } - stage('Heartbeat'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_HEARTBEAT != "false" - } - } - stages { - stage('Heartbeat Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin("Heartbeat oss Windows Unit test", "heartbeat", "build unitTest") - } - } - } - } - stage('Auditbeat oss Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_AUDITBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Auditbeat oss Windows Unit test", "auditbeat", "build unitTest") - } - } - stage('Auditbeat x-pack Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_AUDITBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Auditbeat x-pack Windows", "x-pack/auditbeat", "build unitTest") - } - } - stage('Metricbeat Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_METRICBEAT != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Metricbeat Windows Unit test", "metricbeat", "build unitTest") - } - } - stage('Metricbeat x-pack Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_METRICBEAT_XPACK != "false" && params.windowsTest - } - } - steps { - mageTargetWin("Metricbeat x-pack Windows", "x-pack/metricbeat", "build unitTest") - } - } - stage('Winlogbeat'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_WINLOGBEAT != "false" - } - } - stages { - stage('Winlogbeat Windows'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin("Winlogbeat Windows Unit test", "winlogbeat", "build unitTest") - } - } - } - } - stage('Winlogbeat Windows x-pack'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return params.windowsTest && env.BUILD_WINLOGBEAT_XPACK != "false" - } - } - steps { - mageTargetWin("Winlogbeat x-pack Windows", "x-pack/winlogbeat", "build unitTest") - } - } - stage('Functionbeat'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - // NOTE: commented to run all the windows stages. - //return env.BUILD_FUNCTIONBEAT_XPACK != "false" - } - } - stages { - stage('Functionbeat Windows x-pack'){ - options { skipDefaultCheckout() } - when { - beforeAgent true - expression { - return params.windowsTest - } - } - steps { - mageTargetWin("Functionbeat x-pack Windows Unit test", "x-pack/functionbeat", "build unitTest") - } - } - } - } - } - } - } - post { - always { - runbld() - } - cleanup { - notifyBuildResult(prComment: true) - } - } -} - -def delete() { - dir("${env.BASE_DIR}") { - fixPermissions("${WORKSPACE}") - } - deleteDir() -} - -def fixPermissions(location) { - sh(label: 'Fix permissions', script: """#!/usr/bin/env bash - source ./dev-tools/common.bash - docker_setup - script/fix_permissions.sh ${location}""", returnStatus: true) -} - -def makeTarget(String context, String target, boolean clean = true) { - withGithubNotify(context: "${context}") { - withBeatsEnv(true) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - sh(label: "Make ${target}", script: "make ${target}") - whenTrue(clean) { - fixPermissions("${HOME}") - } - } - } -} - -def mageTarget(String context, String directory, String target) { - withGithubNotify(context: "${context}") { - withBeatsEnv(true) { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMage() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - sh(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") - } - } - } -} - -def mageTargetWin(String context, String directory, String target) { - withGithubNotify(context: "${context}") { - def tasks = [:] - windowsVersions.each { os -> - tasks["${context}-${os}"] = mageTargetWin(context, directory, target, os) - } - parallel(tasks) - } -} - -def mageTargetWin(String context, String directory, String target, String label) { - return { - log(level: 'INFO', text: "context=${context} directory=${directory} target=${target} os=${label}") - def immutable = label.equals('windows-7-32-bit') ? 'windows-immutable-32-bit' : 'windows-immutable' - - // NOTE: skip filebeat with windows-2016/2012-r2 since there are some test failures. - // See https://github.com/elastic/beats/issues/19787 https://github.com/elastic/beats/issues/19641 - if (directory.equals('filebeat') && (label.equals('windows-2016') || label.equals('windows-2012-r2'))) { - log(level: 'WARN', text: "Skipped stage for the 'filebeat' with '${label}' as long as there are test failures to be analysed.") - } else { - node("${immutable} && ${label}"){ - withBeatsEnvWin() { - whenTrue(params.debug) { - dumpFilteredEnvironment() - dumpMageWin() - } - - def verboseFlag = params.debug ? "-v" : "" - dir(directory) { - bat(label: "Mage ${target}", script: "mage ${verboseFlag} ${target}") - } - } - } - } - } -} - -def withBeatsEnv(boolean archive, Closure body) { - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - - withEnv([ - "HOME=${env.WORKSPACE}", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - "MAGEFILE_CACHE=${WORKSPACE}/.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - "PYTHON_ENV=${WORKSPACE}/python-env", - "TEST_TAGS=${env.TEST_TAGS},oracle", - "DOCKER_PULL=0", - ]) { - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - if(isDockerInstalled()){ - dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}") - } - dir("${env.BASE_DIR}") { - installTools() - // TODO (2020-04-07): This is a work-around to fix the Beat generator tests. - // See https://github.com/elastic/beats/issues/17787. - setGitConfig() - try { - if(!params.dry_run){ - body() - } - } finally { - if (archive) { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**/build/TEST*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/build/TEST*.out') - } - } - reportCoverage() - } - } - } -} - -def withBeatsEnvWin(Closure body) { - final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' - final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' - // NOTE: to support Windows 7 32 bits the arch in the go context path is required. - def arch = is32bit() ? '386' : 'amd64' - def goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.${arch}" - - withEnv([ - "HOME=${env.WORKSPACE}", - "DEV_ARCH=${arch}", - "DEV_OS=windows", - "GOPATH=${env.WORKSPACE}", - "GOROOT=${goRoot}", - "PATH=${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};C:\\tools\\mingw64\\bin;${env.PATH}", - "MAGEFILE_CACHE=${env.WORKSPACE}\\.magefile", - "TEST_COVERAGE=true", - "RACE_DETECTOR=true", - ]){ - deleteDir() - unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") - dir("${env.BASE_DIR}"){ - installTools() - try { - if(!params.dry_run){ - body() - } - } finally { - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: "**\\build\\TEST*.xml") - archiveArtifacts(allowEmptyArchive: true, artifacts: '**\\build\\TEST*.out') - } - } - } - } -} - -def installTools() { - def i = 2 // Number of retries - if(isUnix()) { - retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") } - retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") } - retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") } - retry(i) { sh(label: "Install Mage", script: "make mage") } - } else { - retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") } - } -} - -def is32bit(){ - def labels = env.NODE_LABELS - return labels.contains('i386') -} - -def goos(){ - def labels = env.NODE_LABELS - - if (labels.contains('linux')) { - return 'linux' - } else if (labels.contains('windows')) { - return 'windows' - } else if (labels.contains('darwin')) { - return 'darwin' - } - - error("Unhandled OS name in NODE_LABELS: " + labels) -} - -def dumpMage(){ - echo "### MAGE DUMP ###" - sh(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpMageWin(){ - echo "### MAGE DUMP ###" - bat(label: "Dump mage variables", script: "mage dumpVariables") - echo "### END MAGE DUMP ###" -} - -def dumpFilteredEnvironment(){ - echo "### ENV DUMP ###" - echo "PATH: ${env.PATH}" - echo "HOME: ${env.HOME}" - echo "USERPROFILE: ${env.USERPROFILE}" - echo "BUILD_DIR: ${env.BUILD_DIR}" - echo "COVERAGE_DIR: ${env.COVERAGE_DIR}" - echo "BEATS: ${env.BEATS}" - echo "PROJECTS: ${env.PROJECTS}" - echo "PROJECTS_ENV: ${env.PROJECTS_ENV}" - echo "PYTHON_ENV: ${env.PYTHON_ENV}" - echo "PYTHON_EXE: ${env.PYTHON_EXE}" - echo "PYTHON_ENV_EXE: ${env.PYTHON_ENV_EXE}" - echo "VENV_PARAMS: ${env.VENV_PARAMS}" - echo "FIND: ${env.FIND}" - echo "GOLINT: ${env.GOLINT}" - echo "GOLINT_REPO: ${env.GOLINT_REPO}" - echo "REVIEWDOG: ${env.REVIEWDOG}" - echo "REVIEWDOG_OPTIONS: ${env.REVIEWDOG_OPTIONS}" - echo "REVIEWDOG_REPO: ${env.REVIEWDOG_REPO}" - echo "XPACK_SUFFIX: ${env.XPACK_SUFFIX}" - echo "PKG_BUILD_DIR: ${env.PKG_BUILD_DIR}" - echo "PKG_UPLOAD_DIR: ${env.PKG_UPLOAD_DIR}" - echo "COVERAGE_TOOL: ${env.COVERAGE_TOOL}" - echo "COVERAGE_TOOL_REPO: ${env.COVERAGE_TOOL_REPO}" - echo "TESTIFY_TOOL_REPO: ${env.TESTIFY_TOOL_REPO}" - echo "NOW: ${env.NOW}" - echo "GOBUILD_FLAGS: ${env.GOBUILD_FLAGS}" - echo "GOIMPORTS: ${env.GOIMPORTS}" - echo "GOIMPORTS_REPO: ${env.GOIMPORTS_REPO}" - echo "GOIMPORTS_LOCAL_PREFIX: ${env.GOIMPORTS_LOCAL_PREFIX}" - echo "PROCESSES: ${env.PROCESSES}" - echo "TIMEOUT: ${env.TIMEOUT}" - echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}" - echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}" - echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}" - echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}" - echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}" - echo "STRESS_TESTS: ${env.STRESS_TESTS}" - echo "STRESS_TEST_OPTIONS: ${env.STRESS_TEST_OPTIONS}" - echo "TEST_TAGS: ${env.TEST_TAGS}" - echo "GOX_OS: ${env.GOX_OS}" - echo "GOX_OSARCH: ${env.GOX_OSARCH}" - echo "GOX_FLAGS: ${env.GOX_FLAGS}" - echo "TESTING_ENVIRONMENT: ${env.TESTING_ENVIRONMENT}" - echo "BEAT_VERSION: ${env.BEAT_VERSION}" - echo "COMMIT_ID: ${env.COMMIT_ID}" - echo "DOCKER_COMPOSE_PROJECT_NAME: ${env.DOCKER_COMPOSE_PROJECT_NAME}" - echo "DOCKER_COMPOSE: ${env.DOCKER_COMPOSE}" - echo "DOCKER_CACHE: ${env.DOCKER_CACHE}" - echo "GOPACKAGES_COMMA_SEP: ${env.GOPACKAGES_COMMA_SEP}" - echo "PIP_INSTALL_PARAMS: ${env.PIP_INSTALL_PARAMS}" - echo "### END ENV DUMP ###" -} - -def k8sTest(versions){ - versions.each{ v -> - stage("k8s ${v}"){ - withEnv(["K8S_VERSION=${v}", "KIND_VERSION=v0.7.0", "KUBECONFIG=${env.WORKSPACE}/kubecfg"]){ - withGithubNotify(context: "K8s ${v}") { - withBeatsEnv(false) { - sh(label: "Install kind", script: ".ci/scripts/install-kind.sh") - sh(label: "Install kubectl", script: ".ci/scripts/install-kubectl.sh") - sh(label: "Setup kind", script: ".ci/scripts/kind-setup.sh") - sh(label: "Integration tests", script: "MODULE=kubernetes make -C metricbeat integration-tests") - sh(label: "Deploy to kubernetes",script: "make -C deploy/kubernetes test") - sh(label: 'Delete cluster', script: 'kind delete cluster') - } - } - } - } - } -} - -def reportCoverage(){ - catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { - retry(2){ - sh(label: 'Report to Codecov', script: ''' - curl -sSLo codecov https://codecov.io/bash - for i in auditbeat filebeat heartbeat libbeat metricbeat packetbeat winlogbeat journalbeat - do - FILE="${i}/build/coverage/full.cov" - if [ -f "${FILE}" ]; then - bash codecov -f "${FILE}" - fi - done - ''') - } - } -} - -// isChanged treats the patterns as regular expressions. In order to check if -// any file in a directoy is modified use `^/.*`. -def isChanged(patterns){ - return ( - params.runAllStages - || isGitRegionMatch(patterns: patterns, comparator: 'regexp') - ) -} - -def isChangedOSSCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^libbeat/.*", - "^testing/.*", - "^dev-tools/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -def isChangedXPackCode(patterns) { - def allPatterns = [ - "^Jenkinsfile", - "^go.mod", - "^libbeat/.*", - "^dev-tools/.*", - "^testing/.*", - "^x-pack/libbeat/.*", - "^\\.ci/scripts/.*", - ] - allPatterns.addAll(patterns) - return isChanged(allPatterns) -} - -// withCloudTestEnv executes a closure with credentials for cloud test -// environments. -def withCloudTestEnv(Closure body) { - def maskedVars = [] - def testTags = "${env.TEST_TAGS}" - - // AWS - if (params.allCloudTests || params.awsCloudTests) { - testTags = "${testTags},aws" - def aws = getVaultSecret(secret: "${AWS_ACCOUNT_SECRET}").data - if (!aws.containsKey('access_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'access_key'") - } - if (!aws.containsKey('secret_key')) { - error("${AWS_ACCOUNT_SECRET} doesn't contain 'secret_key'") - } - maskedVars.addAll([ - [var: "AWS_REGION", password: params.awsRegion], - [var: "AWS_ACCESS_KEY_ID", password: aws.access_key], - [var: "AWS_SECRET_ACCESS_KEY", password: aws.secret_key], - ]) - } - - withEnv([ - "TEST_TAGS=${testTags}", - ]) { - withEnvMask(vars: maskedVars) { - body() - } - } -} - -def terraformInit(String directory) { - dir(directory) { - sh(label: "Terraform Init on ${directory}", script: "terraform init") - } -} - -def terraformApply(String directory) { - terraformInit(directory) - dir(directory) { - sh(label: "Terraform Apply on ${directory}", script: "terraform apply -auto-approve") - } -} - -// Start testing environment on cloud using terraform. Terraform files are -// stashed so they can be used by other stages. They are also archived in -// case manual cleanup is needed. -// -// Example: -// startCloudTestEnv('x-pack-metricbeat', [ -// [cond: params.awsCloudTests, dir: 'x-pack/metricbeat/module/aws'], -// ]) -// ... -// terraformCleanup('x-pack-metricbeat', 'x-pack/metricbeat') -def startCloudTestEnv(String name, environments = []) { - withCloudTestEnv() { - withBeatsEnv(false) { - def runAll = params.runAllCloudTests - try { - for (environment in environments) { - if (environment.cond || runAll) { - retry(2) { - terraformApply(environment.dir) - } - } - } - } finally { - // Archive terraform states in case manual cleanup is needed. - archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate') - } - stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**') - } - } -} - - -// Looks for all terraform states in directory and runs terraform destroy for them, -// it uses terraform states previously stashed by startCloudTestEnv. -def terraformCleanup(String stashName, String directory) { - stage("Remove cloud scenarios in ${directory}"){ - withCloudTestEnv() { - withBeatsEnv(false) { - unstash("terraform-${stashName}") - retry(2) { - sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}") - } - } - } - } -} - -def loadConfigEnvVars(){ - def empty = [] - env.GO_VERSION = readFile(".go-version").trim() - - withEnv(["HOME=${env.WORKSPACE}"]) { - retry(2) { sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") } - } - - // Libbeat is the core framework of Beats. It has no additional dependencies - // on other projects in the Beats repository. - env.BUILD_LIBBEAT = isChangedOSSCode(empty) - env.BUILD_LIBBEAT_XPACK = isChangedXPackCode(empty) - - // Auditbeat depends on metricbeat as framework, but does not include any of - // the modules from Metricbeat. - // The Auditbeat x-pack build contains all functionality from OSS Auditbeat. - env.BUILD_AUDITBEAT = isChangedOSSCode(getVendorPatterns('auditbeat')) - env.BUILD_AUDITBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/auditbeat')) - - // Dockerlogbeat is a standalone Beat that only relies on libbeat. - env.BUILD_DOCKERLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/dockerlogbeat')) - - // Filebeat depends on libbeat only. - // The Filebeat x-pack build contains all functionality from OSS Filebeat. - env.BUILD_FILEBEAT = isChangedOSSCode(getVendorPatterns('filebeat')) - env.BUILD_FILEBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/filebeat')) - - // Metricbeat depends on libbeat only. - // The Metricbeat x-pack build contains all functionality from OSS Metricbeat. - env.BUILD_METRICBEAT = isChangedOSSCode(getVendorPatterns('metricbeat')) - env.BUILD_METRICBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/metricbeat')) - - // Functionbeat is a standalone beat that depends on libbeat only. - // Functionbeat is available as x-pack build only. - env.BUILD_FUNCTIONBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/functionbeat')) - - // Heartbeat depends on libbeat only. - // The Heartbeat x-pack build contains all functionality from OSS Heartbeat. - env.BUILD_HEARTBEAT = isChangedOSSCode(getVendorPatterns('heartbeat')) - env.BUILD_HEARTBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/heartbeat')) - - // Journalbeat depends on libbeat only. - // The Journalbeat x-pack build contains all functionality from OSS Journalbeat. - env.BUILD_JOURNALBEAT = isChangedOSSCode(getVendorPatterns('journalbeat')) - env.BUILD_JOURNALBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/journalbeat')) - - // Packetbeat depends on libbeat only. - // The Packetbeat x-pack build contains all functionality from OSS Packetbeat. - env.BUILD_PACKETBEAT = isChangedOSSCode(getVendorPatterns('packetbeat')) - env.BUILD_PACKETBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/packetbeat')) - - // Winlogbeat depends on libbeat only. - // The Winlogbeat x-pack build contains all functionality from OSS Winlogbeat. - env.BUILD_WINLOGBEAT = isChangedOSSCode(getVendorPatterns('winlogbeat')) - env.BUILD_WINLOGBEAT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/winlogbeat')) - - // Elastic-agent is a self-contained product, that depends on libbeat only. - // The agent acts as a supervisor for other Beats like Filebeat or Metricbeat. - // The agent is available as x-pack build only. - env.BUILD_ELASTIC_AGENT_XPACK = isChangedXPackCode(getVendorPatterns('x-pack/elastic-agent')) - - // The Kubernetes test use Filebeat and Metricbeat, but only need to be run - // if the deployment scripts have been updated. No Beats specific testing is - // involved. - env.BUILD_KUBERNETES = isChanged(["^deploy/kubernetes/.*"]) - - def generatorPatterns = ['^generator/.*'] - generatorPatterns.addAll(getVendorPatterns('generator/common/beatgen')) - generatorPatterns.addAll(getVendorPatterns('metricbeat/beater')) - env.BUILD_GENERATOR = isChangedOSSCode(generatorPatterns) - - // Skip all the stages for changes only related to the documentation - env.ONLY_DOCS = isDocChangedOnly() - - // Run the ITs by running only if the changeset affects a specific module. - // For such, it's required to look for changes under the module folder and exclude anything else - // such as ascidoc and png files. - env.MODULE = getGitMatchingGroup(pattern: '[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*', exclude: '^(((?!\\/module\\/).)*$|.*\\.asciidoc|.*\\.png)') -} - -/** - This method verifies if the changeset for the current pull request affect only changes related - to documentation, such as asciidoc and png files. -*/ -def isDocChangedOnly(){ - if (params.runAllStages || !env.CHANGE_ID?.trim()) { - log(level: 'INFO', text: 'Speed build for docs only is disabled for branches/tags or when forcing with the runAllStages parameter.') - return 'false' - } else { - log(level: "INFO", text: 'Check if the speed build for docs is enabled.') - return isGitRegionMatch(patterns: ['.*\\.(asciidoc|png)'], shouldMatchAll: true) - } -} - -/** - This method grab the dependencies of a Go module and transform them on regexp -*/ -def getVendorPatterns(beatName){ - def os = goos() - def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" - def output = "" - - withEnv([ - "HOME=${env.WORKSPACE}/${env.BASE_DIR}", - "PATH=${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}", - ]) { - output = sh(label: 'Get vendor dependency patterns', returnStdout: true, script: """ - go list -deps ./${beatName} \ - | grep 'elastic/beats' \ - | sed -e "s#github.com/elastic/beats/v7/##g" \ - | awk '{print "^" \$1 "/.*"}' - """) - } - return output?.split('\n').collect{ item -> item as String } -} - -def setGitConfig(){ - sh(label: 'check git config', script: ''' - if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" - fi - ''') -} - -def isDockerInstalled(){ - return sh(label: 'check for Docker', script: 'command -v docker', returnStatus: true) -} - -def junitAndStore(Map params = [:]){ - junit(params) - // STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds - def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}" - stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true) - stashedTestReports[stageName] = stageName -} - -def runbld() { - catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') { - if (stashedTestReports) { - dir("${env.BASE_DIR}") { - sh(label: 'Prepare workspace context', - script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete') - // Unstash the test reports - stashedTestReports.each { k, v -> - dir(k) { - unstash(v) - } - } - sh(label: 'Process JUnit reports with runbld', - script: '''\ - cat >./runbld-script < - generateStages(project: projectName, changeset: content['changeset']).each { k,v -> - mapParallelTasks["${k}"] = v + if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) { + error 'Pull Request has been configured to be disabled when there is a skip-ci label match' + } else { + content['projects'].each { projectName -> + generateStages(project: projectName, changeset: content['changeset']).each { k,v -> + mapParallelTasks["${k}"] = v + } } + notifyBuildReason() + parallel(mapParallelTasks) } - notifyBuildReason() - parallel(mapParallelTasks) } } } @@ -122,7 +126,7 @@ pipeline { runbld(stashedTestReports: stashedTestReports, project: env.REPO) } cleanup { - notifyBuildResult(prComment: true, slackComment: true) + notifyBuildResult(prComment: true, slackComment: true, slackNotify: (isBranch() || isTag())) } } } @@ -274,8 +278,8 @@ def withBeatsEnv(Map args = [:], Closure body) { // See https://github.com/elastic/beats/issues/17787. sh(label: 'check git config', script: ''' if [ -z "$(git config --get user.email)" ]; then - git config user.email "beatsmachine@users.noreply.github.com" - git config user.name "beatsmachine" + git config --global user.email "beatsmachine@users.noreply.github.com" + git config --global user.name "beatsmachine" fi''') } try { @@ -354,8 +358,13 @@ def archiveTestOutput(Map args = [:]) { } cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py') dir('build') { + if (isUnix()) { + cmd(label: 'Delete folders that are causing exceptions (See JENKINS-58421)', + returnStatus: true, + script: 'rm -rf ve || true; find . -type d -name vendor -exec rm -r {} \\;') + } else { log(level: 'INFO', text: 'Delete folders that are causing exceptions (See JENKINS-58421) is disabled for Windows.') } junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id) - archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts) + tar(file: "test-build-artifacts-${args.id}.tgz", dir: '.', archive: true, allowMissing: true) } catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml index 2278ea93735..f7b21e1cbdf 100644 --- a/Jenkinsfile.yml +++ b/Jenkinsfile.yml @@ -40,10 +40,9 @@ changeset: - "^testing/.*" - "^x-pack/libbeat/.*" -## Proposal -## TBC: This will allow to configure what to do based on the PR configuration disabled: when: - labels: ## Skip the GitHub Pull Request builds if there is a GitHub label match - - "skip-ci" - draft: true ## Skip the GitHub Pull Request builds with Draft PRs. + labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR. + - skip-ci + ## TODO: This will allow to configure what to do based on the PR configuration + draft: true ## Skip the GitHub Pull Request builds with Draft PRs. \ No newline at end of file diff --git a/NOTICE.txt b/NOTICE.txt index 527c1304379..349fe58b3d1 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -4472,6 +4472,192 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/dgraph-io/badger/v2 +Version: v2.2007.3-0.20201012072640-f5a7e0a1c83b +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/badger/v2@v2.2007.3-0.20201012072640-f5a7e0a1c83b/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + -------------------------------------------------------------------------------- Dependency : github.com/digitalocean/go-libvirt Version: v0.0.0-20180301200012-6075ea3c39a1 @@ -5542,11 +5728,11 @@ OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -------------------------------------------------------------------------------- Dependency : github.com/dustin/go-humanize -Version: v0.0.0-20171111073723-bb3d318650d4 +Version: v1.0.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/dustin/go-humanize@v0.0.0-20171111073723-bb3d318650d4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/dustin/go-humanize@v1.0.0/LICENSE: Copyright (c) 2005-2008 Dustin Sallings @@ -13066,11 +13252,11 @@ CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- Dependency : github.com/spf13/cobra -Version: v0.0.3 +Version: v0.0.5 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v0.0.3/LICENSE.txt: +Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v0.0.5/LICENSE.txt: Apache License Version 2.0, January 2004 @@ -13393,224 +13579,256 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/urso/sderr -Version: v0.0.0-20200210124243-c2a16f3d43ec -Licence type (autodetected): Apache-2.0 +Dependency : github.com/ugorji/go/codec +Version: v1.1.8 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +Contents of probable licence file $GOMODCACHE/github.com/ugorji/go/codec@v1.1.8/LICENSE: - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +The MIT License (MIT) - Copyright [yyyy] [name of copyright owner] +Copyright (c) 2012-2015 Ugorji Nwoke. +All rights reserved. - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - http://www.apache.org/licenses/LICENSE-2.0 +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/vmware/govmomi -Version: v0.0.0-20170802214208-2cad15190b41 +Dependency : github.com/urso/sderr +Version: v0.0.0-20200210124243-c2a16f3d43ec Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/vmware/govmomi@v0.0.0-20170802214208-2cad15190b41/LICENSE.txt: - +Contents of probable licence file $GOMODCACHE/github.com/urso/sderr@v0.0.0-20200210124243-c2a16f3d43ec/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/vmware/govmomi +Version: v0.0.0-20170802214208-2cad15190b41 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/vmware/govmomi@v0.0.0-20170802214208-2cad15190b41/LICENSE.txt: + Apache License Version 2.0, January 2004 @@ -19482,6 +19700,43 @@ Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/xgb@v0.0.0 // such litigation is filed. +-------------------------------------------------------------------------------- +Dependency : github.com/DataDog/zstd +Version: v1.4.1 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!data!dog/zstd@v1.4.1/LICENSE: + +Simplified BSD License + +Copyright (c) 2016, Datadog +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/Masterminds/semver Version: v1.4.2 @@ -19565,6 +19820,203 @@ See the License for the specific language governing permissions and limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/OneOfOne/xxhash +Version: v1.2.2 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!one!of!one/xxhash@v1.2.2/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + -------------------------------------------------------------------------------- Dependency : github.com/PuerkitoBio/purell Version: v1.0.0 @@ -19802,6 +20254,377 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------------------- +Dependency : github.com/armon/consul-api +Version: v0.0.0-20180202201655-eb2c6b5be1b6 +Licence type (autodetected): MPL-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/armon/consul-api@v0.0.0-20180202201655-eb2c6b5be1b6/LICENSE: + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + -------------------------------------------------------------------------------- Dependency : github.com/armon/go-radix Version: v1.0.0 @@ -20546,6 +21369,38 @@ Contents of probable licence file $GOMODCACHE/github.com/census-instrumentation/ limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/cespare/xxhash +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/cespare/xxhash@v1.1.0/LICENSE.txt: + +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/chzyer/logex Version: v1.1.10 @@ -22348,378 +23203,13 @@ Contents of probable licence file $GOMODCACHE/github.com/containerd/typeurl@v0.0 -------------------------------------------------------------------------------- -Dependency : github.com/coreos/go-systemd -Version: v0.0.0-20190321100706-95778dfbb74e +Dependency : github.com/coreos/etcd +Version: v3.3.10+incompatible Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/coreos/go-systemd@v0.0.0-20190321100706-95778dfbb74e/LICENSE: - -Apache License -Version 2.0, January 2004 -http://www.apache.org/licenses/ - -TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - -1. Definitions. - -"License" shall mean the terms and conditions for use, reproduction, and -distribution as defined by Sections 1 through 9 of this document. - -"Licensor" shall mean the copyright owner or entity authorized by the copyright -owner that is granting the License. - -"Legal Entity" shall mean the union of the acting entity and all other entities -that control, are controlled by, or are under common control with that entity. -For the purposes of this definition, "control" means (i) the power, direct or -indirect, to cause the direction or management of such entity, whether by -contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the -outstanding shares, or (iii) beneficial ownership of such entity. - -"You" (or "Your") shall mean an individual or Legal Entity exercising -permissions granted by this License. - -"Source" form shall mean the preferred form for making modifications, including -but not limited to software source code, documentation source, and configuration -files. - -"Object" form shall mean any form resulting from mechanical transformation or -translation of a Source form, including but not limited to compiled object code, -generated documentation, and conversions to other media types. - -"Work" shall mean the work of authorship, whether in Source or Object form, made -available under the License, as indicated by a copyright notice that is included -in or attached to the work (an example is provided in the Appendix below). - -"Derivative Works" shall mean any work, whether in Source or Object form, that -is based on (or derived from) the Work and for which the editorial revisions, -annotations, elaborations, or other modifications represent, as a whole, an -original work of authorship. For the purposes of this License, Derivative Works -shall not include works that remain separable from, or merely link (or bind by -name) to the interfaces of, the Work and Derivative Works thereof. - -"Contribution" shall mean any work of authorship, including the original version -of the Work and any modifications or additions to that Work or Derivative Works -thereof, that is intentionally submitted to Licensor for inclusion in the Work -by the copyright owner or by an individual or Legal Entity authorized to submit -on behalf of the copyright owner. For the purposes of this definition, -"submitted" means any form of electronic, verbal, or written communication sent -to the Licensor or its representatives, including but not limited to -communication on electronic mailing lists, source code control systems, and -issue tracking systems that are managed by, or on behalf of, the Licensor for -the purpose of discussing and improving the Work, but excluding communication -that is conspicuously marked or otherwise designated in writing by the copyright -owner as "Not a Contribution." - -"Contributor" shall mean Licensor and any individual or Legal Entity on behalf -of whom a Contribution has been received by Licensor and subsequently -incorporated within the Work. - -2. Grant of Copyright License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable copyright license to reproduce, prepare Derivative Works of, -publicly display, publicly perform, sublicense, and distribute the Work and such -Derivative Works in Source or Object form. - -3. Grant of Patent License. - -Subject to the terms and conditions of this License, each Contributor hereby -grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, -irrevocable (except as stated in this section) patent license to make, have -made, use, offer to sell, sell, import, and otherwise transfer the Work, where -such license applies only to those patent claims licensable by such Contributor -that are necessarily infringed by their Contribution(s) alone or by combination -of their Contribution(s) with the Work to which such Contribution(s) was -submitted. If You institute patent litigation against any entity (including a -cross-claim or counterclaim in a lawsuit) alleging that the Work or a -Contribution incorporated within the Work constitutes direct or contributory -patent infringement, then any patent licenses granted to You under this License -for that Work shall terminate as of the date such litigation is filed. - -4. Redistribution. - -You may reproduce and distribute copies of the Work or Derivative Works thereof -in any medium, with or without modifications, and in Source or Object form, -provided that You meet the following conditions: - -You must give any other recipients of the Work or Derivative Works a copy of -this License; and -You must cause any modified files to carry prominent notices stating that You -changed the files; and -You must retain, in the Source form of any Derivative Works that You distribute, -all copyright, patent, trademark, and attribution notices from the Source form -of the Work, excluding those notices that do not pertain to any part of the -Derivative Works; and -If the Work includes a "NOTICE" text file as part of its distribution, then any -Derivative Works that You distribute must include a readable copy of the -attribution notices contained within such NOTICE file, excluding those notices -that do not pertain to any part of the Derivative Works, in at least one of the -following places: within a NOTICE text file distributed as part of the -Derivative Works; within the Source form or documentation, if provided along -with the Derivative Works; or, within a display generated by the Derivative -Works, if and wherever such third-party notices normally appear. The contents of -the NOTICE file are for informational purposes only and do not modify the -License. You may add Your own attribution notices within Derivative Works that -You distribute, alongside or as an addendum to the NOTICE text from the Work, -provided that such additional attribution notices cannot be construed as -modifying the License. -You may add Your own copyright statement to Your modifications and may provide -additional or different license terms and conditions for use, reproduction, or -distribution of Your modifications, or for any such Derivative Works as a whole, -provided Your use, reproduction, and distribution of the Work otherwise complies -with the conditions stated in this License. - -5. Submission of Contributions. - -Unless You explicitly state otherwise, any Contribution intentionally submitted -for inclusion in the Work by You to the Licensor shall be under the terms and -conditions of this License, without any additional terms or conditions. -Notwithstanding the above, nothing herein shall supersede or modify the terms of -any separate license agreement you may have executed with Licensor regarding -such Contributions. - -6. Trademarks. - -This License does not grant permission to use the trade names, trademarks, -service marks, or product names of the Licensor, except as required for -reasonable and customary use in describing the origin of the Work and -reproducing the content of the NOTICE file. - -7. Disclaimer of Warranty. - -Unless required by applicable law or agreed to in writing, Licensor provides the -Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, -including, without limitation, any warranties or conditions of TITLE, -NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are -solely responsible for determining the appropriateness of using or -redistributing the Work and assume any risks associated with Your exercise of -permissions under this License. - -8. Limitation of Liability. - -In no event and under no legal theory, whether in tort (including negligence), -contract, or otherwise, unless required by applicable law (such as deliberate -and grossly negligent acts) or agreed to in writing, shall any Contributor be -liable to You for damages, including any direct, indirect, special, incidental, -or consequential damages of any character arising as a result of this License or -out of the use or inability to use the Work (including but not limited to -damages for loss of goodwill, work stoppage, computer failure or malfunction, or -any and all other commercial damages or losses), even if such Contributor has -been advised of the possibility of such damages. - -9. Accepting Warranty or Additional Liability. - -While redistributing the Work or Derivative Works thereof, You may choose to -offer, and charge a fee for, acceptance of support, warranty, indemnity, or -other liability obligations and/or rights consistent with this License. However, -in accepting such obligations, You may act only on Your own behalf and on Your -sole responsibility, not on behalf of any other Contributor, and only if You -agree to indemnify, defend, and hold each Contributor harmless for any liability -incurred by, or claims asserted against, such Contributor by reason of your -accepting any such warranty or additional liability. - -END OF TERMS AND CONDITIONS - -APPENDIX: How to apply the Apache License to your work - -To apply the Apache License to your work, attach the following boilerplate -notice, with the fields enclosed by brackets "[]" replaced with your own -identifying information. (Don't include the brackets!) The text should be -enclosed in the appropriate comment syntax for the file format. We also -recommend that a file or class name and description of purpose be included on -the same "printed page" as the copyright notice for easier identification within -third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/cucumber/godog -Version: v0.8.1 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/cucumber/godog@v0.8.1/LICENSE: - -The MIT License (MIT) - -Copyright (c) SmartBear - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/cyphar/filepath-securejoin -Version: v0.2.2 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.2/LICENSE: - -Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. -Copyright (C) 2017 SUSE LLC. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +Contents of probable licence file $GOMODCACHE/github.com/coreos/etcd@v3.3.10+incompatible/LICENSE: --------------------------------------------------------------------------------- -Dependency : github.com/davecgh/go-spew -Version: v1.1.1 -Licence type (autodetected): ISC --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-spew@v1.1.1/LICENSE: - -ISC License - -Copyright (c) 2012-2016 Dave Collins - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/davecgh/go-xdr -Version: v0.0.0-20161123171359-e6a2ba005892 -Licence type (autodetected): ISC --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-xdr@v0.0.0-20161123171359-e6a2ba005892/LICENSE: - -Copyright (c) 2012-2014 Dave Collins - -Permission to use, copy, modify, and distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - --------------------------------------------------------------------------------- -Dependency : github.com/devigned/tab -Version: v0.1.2-0.20190607222403-0c15cf42f9a2 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/devigned/tab@v0.1.2-0.20190607222403-0c15cf42f9a2/LICENSE: - -MIT License - -Copyright (c) 2019 David Justice - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/dgrijalva/jwt-go -Version: v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/dgrijalva/jwt-go@v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible/LICENSE: - -Copyright (c) 2012 Dave Grijalva - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - - --------------------------------------------------------------------------------- -Dependency : github.com/dimchansky/utfbom -Version: v1.1.0 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/dimchansky/utfbom@v1.1.0/LICENSE: Apache License Version 2.0, January 2004 @@ -22901,7 +23391,7 @@ Contents of probable licence file $GOMODCACHE/github.com/dimchansky/utfbom@v1.1. APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -22909,7 +23399,7 @@ Contents of probable licence file $GOMODCACHE/github.com/dimchansky/utfbom@v1.1. same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -22925,45 +23415,15 @@ Contents of probable licence file $GOMODCACHE/github.com/dimchansky/utfbom@v1.1. -------------------------------------------------------------------------------- -Dependency : github.com/dlclark/regexp2 -Version: v1.1.7-0.20171009020623-7632a260cbaf -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/dlclark/regexp2@v1.1.7-0.20171009020623-7632a260cbaf/LICENSE: - -The MIT License (MIT) - -Copyright (c) Doug Clark - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/docker/distribution -Version: v2.7.1+incompatible +Dependency : github.com/coreos/go-etcd +Version: v2.0.0+incompatible Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/docker/distribution@v2.7.1+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/coreos/go-etcd@v2.0.0+incompatible/LICENSE: -Apache License + + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -23143,7 +23603,7 @@ Apache License APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" + boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -23151,7 +23611,7 @@ Apache License same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright {yyyy} {name of copyright owner} + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -23166,19 +23626,18 @@ Apache License limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/docker/go-metrics -Version: v0.0.1 +Dependency : github.com/coreos/go-semver +Version: v0.2.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/docker/go-metrics@v0.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/coreos/go-semver@v0.2.0/LICENSE: Apache License Version 2.0, January 2004 - https://www.apache.org/licenses/ + http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -23353,13 +23812,24 @@ Contents of probable licence file $GOMODCACHE/github.com/docker/go-metrics@v0.0. END OF TERMS AND CONDITIONS - Copyright 2013-2016 Docker, Inc. + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -23369,198 +23839,198 @@ Contents of probable licence file $GOMODCACHE/github.com/docker/go-metrics@v0.0. -------------------------------------------------------------------------------- -Dependency : github.com/docker/spdystream -Version: v0.0.0-20160310174837-449fdfce4d96 +Dependency : github.com/coreos/go-systemd +Version: v0.0.0-20190321100706-95778dfbb74e Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/docker/spdystream@v0.0.0-20160310174837-449fdfce4d96/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/coreos/go-systemd@v0.0.0-20190321100706-95778dfbb74e/LICENSE: +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +1. Definitions. - 1. Definitions. +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +2. Grant of Copyright License. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +3. Grant of Patent License. - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and +4. Redistribution. - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. +5. Submission of Contributions. - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. +6. Trademarks. - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. +7. Disclaimer of Warranty. - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. +8. Limitation of Liability. - END OF TERMS AND CONDITIONS +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. - Copyright 2014-2015 Docker, Inc. +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -23570,16 +24040,16 @@ Contents of probable licence file $GOMODCACHE/github.com/docker/spdystream@v0.0. -------------------------------------------------------------------------------- -Dependency : github.com/eapache/go-resiliency -Version: v1.2.0 +Dependency : github.com/cpuguy83/go-md2man +Version: v1.0.10 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cpuguy83/go-md2man@v1.0.10/LICENSE.md: The MIT License (MIT) -Copyright (c) 2014 Evan Huus +Copyright (c) 2014 Brian Goff Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23600,18 +24070,17 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -------------------------------------------------------------------------------- -Dependency : github.com/eapache/go-xerial-snappy -Version: v0.0.0-20180814174437-776d5712da21 +Dependency : github.com/cucumber/godog +Version: v0.8.1 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/eapache/go-xerial-snappy@v0.0.0-20180814174437-776d5712da21/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/cucumber/godog@v0.8.1/LICENSE: The MIT License (MIT) -Copyright (c) 2016 Evan Huus +Copyright (c) SmartBear Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23633,16 +24102,101 @@ SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/eapache/queue -Version: v1.1.0 +Dependency : github.com/cyphar/filepath-securejoin +Version: v0.2.2 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/cyphar/filepath-securejoin@v0.2.2/LICENSE: + +Copyright (C) 2014-2015 Docker Inc & Go Authors. All rights reserved. +Copyright (C) 2017 SUSE LLC. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/davecgh/go-spew +Version: v1.1.1 +Licence type (autodetected): ISC +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-spew@v1.1.1/LICENSE: + +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/davecgh/go-xdr +Version: v0.0.0-20161123171359-e6a2ba005892 +Licence type (autodetected): ISC +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/davecgh/go-xdr@v0.0.0-20161123171359-e6a2ba005892/LICENSE: + +Copyright (c) 2012-2014 Dave Collins + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +-------------------------------------------------------------------------------- +Dependency : github.com/devigned/tab +Version: v0.1.2-0.20190607222403-0c15cf42f9a2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/eapache/queue@v1.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/devigned/tab@v0.1.2-0.20190607222403-0c15cf42f9a2/LICENSE: -The MIT License (MIT) +MIT License -Copyright (c) 2014 Evan Huus +Copyright (c) 2019 David Justice Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -23662,14 +24216,14 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + -------------------------------------------------------------------------------- -Dependency : github.com/elastic/go-windows -Version: v1.0.1 +Dependency : github.com/dgraph-io/ristretto +Version: v0.0.3-0.20200630154024-f66de99634de Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: - +Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/ristretto@v0.0.3-0.20200630154024-f66de99634de/LICENSE: Apache License Version 2.0, January 2004 @@ -23848,107 +24402,65 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0 END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. +-------------------------------------------------------------------------------- +Dependency : github.com/dgrijalva/jwt-go +Version: v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- - Copyright [yyyy] [name of copyright owner] +Contents of probable licence file $GOMODCACHE/github.com/dgrijalva/jwt-go@v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible/LICENSE: - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at +Copyright (c) 2012 Dave Grijalva - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/elazarl/goproxy -Version: v0.0.0-20180725130230-947c36da3153 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elazarl/goproxy@v0.0.0-20180725130230-947c36da3153/LICENSE: - -Copyright (c) 2012 Elazar Leibovich. All rights reserved. +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Elazar Leibovich. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/emicklei/go-restful -Version: v0.0.0-20170410110728-ff4f55a20633 +Dependency : github.com/dgryski/go-farm +Version: v0.0.0-20190423205320-6a90982ecee2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/emicklei/go-restful@v0.0.0-20170410110728-ff4f55a20633/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/dgryski/go-farm@v0.0.0-20190423205320-6a90982ecee2/LICENSE: -Copyright (c) 2012,2013 Ernest Micklei +As this is a highly derivative work, I have placed it under the same license as the original implementation: -MIT License +Copyright (c) 2014-2017 Damian Gryski +Copyright (c) 2016-2017 Nicola Asuni - Tecnick.com -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/envoyproxy/go-control-plane -Version: v0.9.4 +Dependency : github.com/dimchansky/utfbom +Version: v1.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.9.4/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/dimchansky/utfbom@v1.1.0/LICENSE: Apache License Version 2.0, January 2004 @@ -24154,15 +24666,45 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-p -------------------------------------------------------------------------------- -Dependency : github.com/envoyproxy/protoc-gen-validate -Version: v0.1.0 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/dlclark/regexp2 +Version: v1.1.7-0.20171009020623-7632a260cbaf +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-validate@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/dlclark/regexp2@v1.1.7-0.20171009020623-7632a260cbaf/LICENSE: + +The MIT License (MIT) +Copyright (c) Doug Clark - Apache License +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/docker/distribution +Version: v2.7.1+incompatible +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/docker/distribution@v2.7.1+incompatible/LICENSE: + +Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -24342,7 +24884,7 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-v APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -24350,7 +24892,7 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-v same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24365,281 +24907,19 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-v limitations under the License. --------------------------------------------------------------------------------- -Dependency : github.com/evanphx/json-patch -Version: v4.2.0+incompatible -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/evanphx/json-patch@v4.2.0+incompatible/LICENSE: - -Copyright (c) 2014, Evan Phoenix -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. -* Neither the name of the Evan Phoenix nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/fortytw2/leaktest -Version: v1.3.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/fortytw2/leaktest@v1.3.0/LICENSE: - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/frankban/quicktest -Version: v1.7.2 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: - -MIT License - -Copyright (c) 2017 Canonical Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/ghodss/yaml -Version: v1.0.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/ghodss/yaml@v1.0.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2014 Sam Ghods - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/go-gl/glfw/v3.3/glfw -Version: v0.0.0-20191125211704-12ad95a8df72 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/go-gl/glfw/v3.3/glfw@v0.0.0-20191125211704-12ad95a8df72/LICENSE: - -Copyright (c) 2012 The glfw3-go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/go-kit/kit -Version: v0.9.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/go-kit/kit@v0.9.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Peter Bourgon - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - - --------------------------------------------------------------------------------- -Dependency : github.com/go-logfmt/logfmt -Version: v0.4.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/go-logfmt/logfmt@v0.4.0/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 go-logfmt - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - -------------------------------------------------------------------------------- -Dependency : github.com/go-logr/logr -Version: v0.1.0 +Dependency : github.com/docker/go-metrics +Version: v0.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-logr/logr@v0.1.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/docker/go-metrics@v0.0.1/LICENSE: + Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -24814,24 +25094,13 @@ Contents of probable licence file $GOMODCACHE/github.com/go-logr/logr@v0.1.0/LIC END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} + Copyright 2013-2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -24841,73 +25110,12 @@ Contents of probable licence file $GOMODCACHE/github.com/go-logr/logr@v0.1.0/LIC -------------------------------------------------------------------------------- -Dependency : github.com/go-martini/martini -Version: v0.0.0-20170121215854-22fa46961aab -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/go-martini/martini@v0.0.0-20170121215854-22fa46961aab/LICENSE: - -The MIT License (MIT) - -Copyright (c) 2015 Jeremy Saenz - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/go-ole/go-ole -Version: v1.2.5-0.20190920104607-14974a1cf647 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/go-ole/go-ole@v1.2.5-0.20190920104607-14974a1cf647/LICENSE: - -The MIT License (MIT) - -Copyright © 2013-2017 Yasuhiro Matsumoto, - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the “Software”), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/go-openapi/jsonpointer -Version: v0.0.0-20160704185906-46af16f9f7b1 +Dependency : github.com/docker/spdystream +Version: v0.0.0-20160310174837-449fdfce4d96 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonpointer@v0.0.0-20160704185906-46af16f9f7b1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/docker/spdystream@v0.0.0-20160310174837-449fdfce4d96/LICENSE: Apache License @@ -25087,18 +25295,7 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonpointer@ END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] + Copyright 2014-2015 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25114,19 +25311,112 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonpointer@ -------------------------------------------------------------------------------- -Dependency : github.com/go-openapi/jsonreference -Version: v0.0.0-20160704190145-13c6e3589ad9 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/eapache/go-resiliency +Version: v1.2.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonreference@v0.0.0-20160704190145-13c6e3589ad9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/eapache/go-resiliency@v1.2.0/LICENSE: +The MIT License (MIT) - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Copyright (c) 2014 Evan Huus - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/eapache/go-xerial-snappy +Version: v0.0.0-20180814174437-776d5712da21 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/eapache/go-xerial-snappy@v0.0.0-20180814174437-776d5712da21/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2016 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/eapache/queue +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/eapache/queue@v1.1.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +Dependency : github.com/elastic/go-windows +Version: v1.0.1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/elastic/go-windows@v1.0.1/LICENSE.txt: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. @@ -25326,13 +25616,80 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonreferenc -------------------------------------------------------------------------------- -Dependency : github.com/go-openapi/spec -Version: v0.0.0-20160808142527-6aced65f8501 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/elazarl/goproxy +Version: v0.0.0-20180725130230-947c36da3153 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-openapi/spec@v0.0.0-20160808142527-6aced65f8501/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elazarl/goproxy@v0.0.0-20180725130230-947c36da3153/LICENSE: +Copyright (c) 2012 Elazar Leibovich. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Elazar Leibovich. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/emicklei/go-restful +Version: v0.0.0-20170410110728-ff4f55a20633 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/emicklei/go-restful@v0.0.0-20170410110728-ff4f55a20633/LICENSE: + +Copyright (c) 2012,2013 Ernest Micklei + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- +Dependency : github.com/envoyproxy/go-control-plane +Version: v0.9.4 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/go-control-plane@v0.9.4/LICENSE: Apache License Version 2.0, January 2004 @@ -25514,7 +25871,7 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/spec@v0.0.0- APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -25522,7 +25879,7 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/spec@v0.0.0- same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -25538,12 +25895,12 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/spec@v0.0.0- -------------------------------------------------------------------------------- -Dependency : github.com/go-openapi/swag -Version: v0.0.0-20160704191624-1d0bd113de87 +Dependency : github.com/envoyproxy/protoc-gen-validate +Version: v0.1.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-openapi/swag@v0.0.0-20160704191624-1d0bd113de87/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-validate@v0.1.0/LICENSE: Apache License @@ -25750,16 +26107,50 @@ Contents of probable licence file $GOMODCACHE/github.com/go-openapi/swag@v0.0.0- -------------------------------------------------------------------------------- -Dependency : github.com/go-sourcemap/sourcemap -Version: v2.1.2+incompatible -Licence type (autodetected): BSD-2-Clause +Dependency : github.com/evanphx/json-patch +Version: v4.2.0+incompatible +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-sourcemap/sourcemap@v2.1.2+incompatible/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/evanphx/json-patch@v4.2.0+incompatible/LICENSE: -Copyright (c) 2016 The github.com/go-sourcemap/sourcemap Contributors. +Copyright (c) 2014, Evan Phoenix All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/fortytw2/leaktest +Version: v1.3.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/fortytw2/leaktest@v1.3.0/LICENSE: + +Copyright (c) 2012 The Go Authors. All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -25770,6 +26161,9 @@ notice, this list of conditions and the following disclaimer. copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -25785,16 +26179,16 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/go-stack/stack -Version: v1.8.0 +Dependency : github.com/frankban/quicktest +Version: v1.7.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/go-stack/stack@v1.8.0/LICENSE.md: +Contents of probable licence file $GOMODCACHE/github.com/frankban/quicktest@v1.7.2/LICENSE: -The MIT License (MIT) +MIT License -Copyright (c) 2014 Chris Hines +Copyright (c) 2017 Canonical Ltd. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25816,16 +26210,16 @@ SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/gobuffalo/here -Version: v0.6.0 +Dependency : github.com/ghodss/yaml +Version: v1.0.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/gobuffalo/here@v0.6.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/ghodss/yaml@v1.0.0/LICENSE: The MIT License (MIT) -Copyright (c) 2019 Mark Bates +Copyright (c) 2014 Sam Ghods Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -25846,48 +26240,1395 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- -Dependency : github.com/godbus/dbus/v5 -Version: v5.0.3 -Licence type (autodetected): BSD-2-Clause +Dependency : github.com/go-gl/glfw/v3.3/glfw +Version: v0.0.0-20191125211704-12ad95a8df72 +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/godbus/dbus/v5@v5.0.3/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/go-gl/glfw/v3.3/glfw@v0.0.0-20191125211704-12ad95a8df72/LICENSE: -Copyright (c) 2013, Georg Reinke (), Google -All rights reserved. +Copyright (c) 2012 The glfw3-go Authors. All rights reserved. Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions -are met: - -1. Redistributions of source code must retain the above copyright notice, -this list of conditions and the following disclaimer. +modification, are permitted provided that the following conditions are +met: -2. Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/golang-sql/civil -Version: v0.0.0-20190719163853-cb61b32ac6fe -Licence type (autodetected): Apache-2.0 +Dependency : github.com/go-kit/kit +Version: v0.9.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/golang-sql/civil@v0.0.0-20190719163853-cb61b32ac6fe/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/go-kit/kit@v0.9.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Peter Bourgon + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-logfmt/logfmt +Version: v0.4.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-logfmt/logfmt@v0.4.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 go-logfmt + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-logr/logr +Version: v0.1.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-logr/logr@v0.1.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-martini/martini +Version: v0.0.0-20170121215854-22fa46961aab +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-martini/martini@v0.0.0-20170121215854-22fa46961aab/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2015 Jeremy Saenz + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-ole/go-ole +Version: v1.2.5-0.20190920104607-14974a1cf647 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-ole/go-ole@v1.2.5-0.20190920104607-14974a1cf647/LICENSE: + +The MIT License (MIT) + +Copyright © 2013-2017 Yasuhiro Matsumoto, + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-openapi/jsonpointer +Version: v0.0.0-20160704185906-46af16f9f7b1 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonpointer@v0.0.0-20160704185906-46af16f9f7b1/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-openapi/jsonreference +Version: v0.0.0-20160704190145-13c6e3589ad9 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-openapi/jsonreference@v0.0.0-20160704190145-13c6e3589ad9/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-openapi/spec +Version: v0.0.0-20160808142527-6aced65f8501 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-openapi/spec@v0.0.0-20160808142527-6aced65f8501/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-openapi/swag +Version: v0.0.0-20160704191624-1d0bd113de87 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-openapi/swag@v0.0.0-20160704191624-1d0bd113de87/LICENSE: + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-sourcemap/sourcemap +Version: v2.1.2+incompatible +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-sourcemap/sourcemap@v2.1.2+incompatible/LICENSE: + +Copyright (c) 2016 The github.com/go-sourcemap/sourcemap Contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/go-stack/stack +Version: v1.8.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/go-stack/stack@v1.8.0/LICENSE.md: + +The MIT License (MIT) + +Copyright (c) 2014 Chris Hines + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/gobuffalo/here +Version: v0.6.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/gobuffalo/here@v0.6.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2019 Mark Bates + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/godbus/dbus/v5 +Version: v5.0.3 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/godbus/dbus/v5@v5.0.3/LICENSE: + +Copyright (c) 2013, Georg Reinke (), Google +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/golang-sql/civil +Version: v0.0.0-20190719163853-cb61b32ac6fe +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/golang-sql/civil@v0.0.0-20190719163853-cb61b32ac6fe/LICENSE: Apache License @@ -30822,6 +32563,370 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice the Mozilla Public License, v. 2.0. +-------------------------------------------------------------------------------- +Dependency : github.com/hashicorp/hcl +Version: v1.0.0 +Licence type (autodetected): MPL-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/hashicorp/hcl@v1.0.0/LICENSE: + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + + -------------------------------------------------------------------------------- Dependency : github.com/haya14busa/go-actions-toolkit Version: v0.0.0-20200105081403-ca0307860f01 @@ -31829,6 +33934,41 @@ Contents of probable licence file $GOMODCACHE/github.com/kylelemons/godebug@v1.1 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/magiconair/properties +Version: v1.8.0 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/magiconair/properties@v1.8.0/LICENSE: + +goproperties - properties file decoder for Go + +Copyright (c) 2013-2018 - Frank Schroeder + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/mailru/easyjson Version: v0.7.1 @@ -32693,7 +34833,257 @@ Contents of probable licence file $GOMODCACHE/github.com/modern-go/reflect2@v1.0 APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +-------------------------------------------------------------------------------- +Dependency : github.com/morikuni/aec +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/morikuni/aec@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2016 Taihei Morikuni + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/munnerz/goautoneg +Version: v0.0.0-20120707110453-a547fc61f48d +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +No licence file provided. + +-------------------------------------------------------------------------------- +Dependency : github.com/mwitkow/go-conntrack +Version: v0.0.0-20161129095857-cc309e4a2223 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/mwitkow/go-conntrack@v0.0.0-20161129095857-cc309e4a2223/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -32701,7 +35091,7 @@ Contents of probable licence file $GOMODCACHE/github.com/modern-go/reflect2@v1.0 same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -32717,55 +35107,116 @@ Contents of probable licence file $GOMODCACHE/github.com/modern-go/reflect2@v1.0 -------------------------------------------------------------------------------- -Dependency : github.com/morikuni/aec -Version: v1.0.0 -Licence type (autodetected): MIT +Dependency : github.com/mxk/go-flowrate +Version: v0.0.0-20140419014527-cca7078d478f +Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/morikuni/aec@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/mxk/go-flowrate@v0.0.0-20140419014527-cca7078d478f/LICENSE: -The MIT License (MIT) +Copyright (c) 2014 The Go-FlowRate Authors. All rights reserved. -Copyright (c) 2016 Taihei Morikuni +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + * Neither the name of the go-flowrate project nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/munnerz/goautoneg -Version: v0.0.0-20120707110453-a547fc61f48d -Licence type (autodetected): BSD-3-Clause +Dependency : github.com/onsi/ginkgo +Version: v1.11.0 +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -No licence file provided. +Contents of probable licence file $GOMODCACHE/github.com/onsi/ginkgo@v1.11.0/LICENSE: + +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + -------------------------------------------------------------------------------- -Dependency : github.com/mwitkow/go-conntrack -Version: v0.0.0-20161129095857-cc309e4a2223 +Dependency : github.com/onsi/gomega +Version: v1.7.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/onsi/gomega@v1.7.0/LICENSE: + +Copyright (c) 2013-2014 Onsi Fakhouri + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/opencontainers/go-digest +Version: v1.0.0-rc1.0.20190228220655-ac19fd6e7483 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/mwitkow/go-conntrack@v0.0.0-20161129095857-cc309e4a2223/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/opencontainers/go-digest@v1.0.0-rc1.0.20190228220655-ac19fd6e7483/LICENSE: + Apache License Version 2.0, January 2004 - http://www.apache.org/licenses/ + https://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -32940,24 +35391,13 @@ Contents of probable licence file $GOMODCACHE/github.com/mwitkow/go-conntrack@v0 END OF TERMS AND CONDITIONS - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} + Copyright 2016 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -32967,116 +35407,17 @@ Contents of probable licence file $GOMODCACHE/github.com/mwitkow/go-conntrack@v0 -------------------------------------------------------------------------------- -Dependency : github.com/mxk/go-flowrate -Version: v0.0.0-20140419014527-cca7078d478f -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/mxk/go-flowrate@v0.0.0-20140419014527-cca7078d478f/LICENSE: - -Copyright (c) 2014 The Go-FlowRate Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the - distribution. - - * Neither the name of the go-flowrate project nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/onsi/ginkgo -Version: v1.11.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/onsi/ginkgo@v1.11.0/LICENSE: - -Copyright (c) 2013-2014 Onsi Fakhouri - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/onsi/gomega -Version: v1.7.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/onsi/gomega@v1.7.0/LICENSE: - -Copyright (c) 2013-2014 Onsi Fakhouri - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/opencontainers/go-digest -Version: v1.0.0-rc1.0.20190228220655-ac19fd6e7483 +Dependency : github.com/opencontainers/image-spec +Version: v1.0.2-0.20190823105129-775207bd45b6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/opencontainers/go-digest@v1.0.0-rc1.0.20190228220655-ac19fd6e7483/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-spec@v1.0.2-0.20190823105129-775207bd45b6/LICENSE: Apache License Version 2.0, January 2004 - https://www.apache.org/licenses/ + http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION @@ -33251,13 +35592,13 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/go-diges END OF TERMS AND CONDITIONS - Copyright 2016 Docker, Inc. + Copyright 2016 The Linux Foundation. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - https://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -33267,12 +35608,12 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/go-diges -------------------------------------------------------------------------------- -Dependency : github.com/opencontainers/image-spec -Version: v1.0.2-0.20190823105129-775207bd45b6 +Dependency : github.com/opencontainers/runc +Version: v1.0.0-rc9 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-spec@v1.0.2-0.20190823105129-775207bd45b6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runc@v1.0.0-rc9/LICENSE: Apache License @@ -33452,7 +35793,7 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp END OF TERMS AND CONDITIONS - Copyright 2016 The Linux Foundation. + Copyright 2014 Docker, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33468,12 +35809,12 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/image-sp -------------------------------------------------------------------------------- -Dependency : github.com/opencontainers/runc -Version: v1.0.0-rc9 +Dependency : github.com/opencontainers/runtime-spec +Version: v1.0.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runc@v1.0.0-rc9/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-spec@v1.0.1/LICENSE: Apache License @@ -33653,7 +35994,7 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runc@v1. END OF TERMS AND CONDITIONS - Copyright 2014 Docker, Inc. + Copyright 2015 The Linux Foundation. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -33669,12 +36010,12 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runc@v1. -------------------------------------------------------------------------------- -Dependency : github.com/opencontainers/runtime-spec -Version: v1.0.1 +Dependency : github.com/opencontainers/runtime-tools +Version: v0.0.0-20181011054405-1d69bd0f9c39 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-spec@v1.0.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-tools@v0.0.0-20181011054405-1d69bd0f9c39/LICENSE: Apache License @@ -33870,12 +36211,60 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime- -------------------------------------------------------------------------------- -Dependency : github.com/opencontainers/runtime-tools -Version: v0.0.0-20181011054405-1d69bd0f9c39 +Dependency : github.com/otiai10/curr +Version: v1.0.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/curr@v1.0.0/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2020 Hiromu Ochiai + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/otiai10/mint +Version: v1.3.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE: + +Copyright 2017 otiai10 (Hiromu OCHIAI) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/oxtoacart/bpool +Version: v0.0.0-20150712133111-4e1c5567d7c2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime-tools@v0.0.0-20181011054405-1d69bd0f9c39/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0-20150712133111-4e1c5567d7c2/LICENSE: Apache License @@ -34055,7 +36444,18 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime- END OF TERMS AND CONDITIONS - Copyright 2015 The Linux Foundation. + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 Percy Wegmann Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34071,16 +36471,45 @@ Contents of probable licence file $GOMODCACHE/github.com/opencontainers/runtime- -------------------------------------------------------------------------------- -Dependency : github.com/otiai10/curr -Version: v1.0.0 +Dependency : github.com/pelletier/go-toml +Version: v1.2.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/otiai10/curr@v1.0.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/pelletier/go-toml@v1.2.0/LICENSE: The MIT License (MIT) -Copyright (c) 2020 Hiromu Ochiai +Copyright (c) 2013 - 2017 Thomas Pelletier, Eric Anderton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/peterbourgon/diskv +Version: v2.0.1+incompatible +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/peterbourgon/diskv@v2.0.1+incompatible/LICENSE: + +Copyright (c) 2011-2012 Peter Bourgon Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -34102,30 +36531,81 @@ THE SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/otiai10/mint -Version: v1.3.1 +Dependency : github.com/pierrec/lz4 +Version: v2.4.1+incompatible +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.4.1+incompatible/LICENSE: + +Copyright (c) 2015, Pierre Curto +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of xxHash nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/poy/eachers +Version: v0.0.0-20181020210610-23942921fe77 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/otiai10/mint@v1.3.1/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/poy/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md: -Copyright 2017 otiai10 (Hiromu OCHIAI) +The MIT License (MIT) -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: +Copyright (c) 2016 Andrew Poydence -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. -------------------------------------------------------------------------------- -Dependency : github.com/oxtoacart/bpool -Version: v0.0.0-20150712133111-4e1c5567d7c2 +Dependency : github.com/prometheus/client_golang +Version: v1.1.1-0.20190913103102-20428fa0bffc Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0-20150712133111-4e1c5567d7c2/LICENSE: - +Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golang@v1.1.1-0.20190913103102-20428fa0bffc/LICENSE: Apache License Version 2.0, January 2004 @@ -34262,179 +36742,82 @@ Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0- this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2014 Percy Wegmann - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - --------------------------------------------------------------------------------- -Dependency : github.com/peterbourgon/diskv -Version: v2.0.1+incompatible -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/peterbourgon/diskv@v2.0.1+incompatible/LICENSE: - -Copyright (c) 2011-2012 Peter Bourgon - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/pierrec/lz4 -Version: v2.4.1+incompatible -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/pierrec/lz4@v2.4.1+incompatible/LICENSE: - -Copyright (c) 2015, Pierre Curto -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + with Licensor regarding such Contributions. -* Neither the name of xxHash nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. --------------------------------------------------------------------------------- -Dependency : github.com/poy/eachers -Version: v0.0.0-20181020210610-23942921fe77 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- + END OF TERMS AND CONDITIONS -Contents of probable licence file $GOMODCACHE/github.com/poy/eachers@v0.0.0-20181020210610-23942921fe77/LICENSE.md: + APPENDIX: How to apply the Apache License to your work. -The MIT License (MIT) + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. -Copyright (c) 2016 Andrew Poydence + Copyright [yyyy] [name of copyright owner] -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + http://www.apache.org/licenses/LICENSE-2.0 -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. -------------------------------------------------------------------------------- -Dependency : github.com/prometheus/client_golang -Version: v1.1.1-0.20190913103102-20428fa0bffc +Dependency : github.com/rakyll/statik +Version: v0.1.6 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golang@v1.1.1-0.20190913103102-20428fa0bffc/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/rakyll/statik@v0.1.6/LICENSE: + Apache License Version 2.0, January 2004 @@ -34624,7 +37007,7 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golan same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2014 Google Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34640,13 +37023,163 @@ Contents of probable licence file $GOMODCACHE/github.com/prometheus/client_golan -------------------------------------------------------------------------------- -Dependency : github.com/rakyll/statik -Version: v0.1.6 -Licence type (autodetected): Apache-2.0 +Dependency : github.com/reviewdog/errorformat +Version: v0.0.0-20200109134752-8983be9bc7dd +Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rakyll/statik@v0.1.6/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/reviewdog/errorformat@v0.0.0-20200109134752-8983be9bc7dd/LICENSE: + +MIT License + +Copyright (c) 2016 haya14busa + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/rogpeppe/fastuuid +Version: v1.2.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/fastuuid@v1.2.0/LICENSE: + +Copyright © 2014, Roger Peppe +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of this project nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/rogpeppe/go-internal +Version: v1.3.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.3.0/LICENSE: + +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/russross/blackfriday +Version: v1.5.2 +Licence type (autodetected): BSD-2-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/russross/blackfriday@v1.5.2/LICENSE.txt: + +Blackfriday is distributed under the Simplified BSD License: + +> Copyright © 2011 Russ Ross +> All rights reserved. +> +> Redistribution and use in source and binary forms, with or without +> modification, are permitted provided that the following conditions +> are met: +> +> 1. Redistributions of source code must retain the above copyright +> notice, this list of conditions and the following disclaimer. +> +> 2. Redistributions in binary form must reproduce the above +> copyright notice, this list of conditions and the following +> disclaimer in the documentation and/or other materials provided with +> the distribution. +> +> THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +> "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +> LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +> FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +> COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +> INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +> BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +> LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +> CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +> LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +> ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +> POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------------------------------- +Dependency : github.com/samuel/go-parser +Version: v0.0.0-20130731160455-ca8abbf65d0e +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +No licence file provided. + +-------------------------------------------------------------------------------- +Dependency : github.com/sanathkr/go-yaml +Version: v0.0.0-20170819195128-ed9d249f429b +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/sanathkr/go-yaml@v0.0.0-20170819195128-ed9d249f429b/LICENSE: Apache License Version 2.0, January 2004 @@ -34828,7 +37361,7 @@ Contents of probable licence file $GOMODCACHE/github.com/rakyll/statik@v0.1.6/LI APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -34836,7 +37369,7 @@ Contents of probable licence file $GOMODCACHE/github.com/rakyll/statik@v0.1.6/LI same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2014 Google Inc. + Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -34852,16 +37385,16 @@ Contents of probable licence file $GOMODCACHE/github.com/rakyll/statik@v0.1.6/LI -------------------------------------------------------------------------------- -Dependency : github.com/reviewdog/errorformat -Version: v0.0.0-20200109134752-8983be9bc7dd +Dependency : github.com/sanathkr/yaml +Version: v1.0.1-0.20170819201035-0056894fa522 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/reviewdog/errorformat@v0.0.0-20200109134752-8983be9bc7dd/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/sanathkr/yaml@v1.0.1-0.20170819201035-0056894fa522/LICENSE: -MIT License +The MIT License (MIT) -Copyright (c) 2016 haya14busa +Copyright (c) 2014 Sam Ghods Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -34882,51 +37415,44 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. --------------------------------------------------------------------------------- -Dependency : github.com/rogpeppe/fastuuid -Version: v1.2.0 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/fastuuid@v1.2.0/LICENSE: - -Copyright © 2014, Roger Peppe -All rights reserved. +Copyright (c) 2012 The Go Authors. All rights reserved. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - * Neither the name of this project nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Dependency : github.com/rogpeppe/go-internal -Version: v1.3.0 +Dependency : github.com/santhosh-tekuri/jsonschema +Version: v1.2.4 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/rogpeppe/go-internal@v1.3.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/santhosh-tekuri/jsonschema@v1.2.4/LICENSE: -Copyright (c) 2018 The Go Authors. All rights reserved. +Copyright (c) 2017 Santhosh Kumar Tekuri. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -34954,24 +37480,206 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/satori/go.uuid +Version: v1.2.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/satori/go.uuid@v1.2.0/LICENSE: + +Copyright (C) 2013-2018 by Maxim Bublis + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + -------------------------------------------------------------------------------- -Dependency : github.com/samuel/go-parser -Version: v0.0.0-20130731160455-ca8abbf65d0e +Dependency : github.com/sergi/go-diff +Version: v1.1.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/sergi/go-diff@v1.1.0/LICENSE: + +Copyright (c) 2012-2016 The go-diff Authors. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + + +-------------------------------------------------------------------------------- +Dependency : github.com/sirupsen/logrus +Version: v1.4.2 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/sirupsen/logrus@v1.4.2/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 Simon Eskildsen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Dependency : github.com/smartystreets/assertions +Version: v0.0.0-20180927180507-b2de0cb4f26d +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/smartystreets/assertions@v0.0.0-20180927180507-b2de0cb4f26d/LICENSE.md: + +Copyright (c) 2016 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + + +-------------------------------------------------------------------------------- +Dependency : github.com/smartystreets/goconvey +Version: v0.0.0-20190330032615-68dc04aab96a +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/smartystreets/goconvey@v0.0.0-20190330032615-68dc04aab96a/LICENSE.md: + +Copyright (c) 2016 SmartyStreets, LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +NOTE: Various optional and subordinate components carry their own licensing +requirements and restrictions. Use of those components is subject to the terms +and conditions outlined the respective license of each component. + + +-------------------------------------------------------------------------------- +Dependency : github.com/spaolacci/murmur3 +Version: v1.1.0 Licence type (autodetected): BSD-3-Clause -------------------------------------------------------------------------------- -No licence file provided. +Contents of probable licence file $GOMODCACHE/github.com/spaolacci/murmur3@v1.1.0/LICENSE: + +Copyright 2013, Sébastien Paolacci. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the library nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + -------------------------------------------------------------------------------- -Dependency : github.com/sanathkr/go-yaml -Version: v0.0.0-20170819195128-ed9d249f429b +Dependency : github.com/spf13/afero +Version: v1.2.2 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/sanathkr/go-yaml@v0.0.0-20170819195128-ed9d249f429b/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/spf13/afero@v1.2.2/LICENSE.txt: - Apache License + Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -35146,45 +37854,18 @@ Contents of probable licence file $GOMODCACHE/github.com/sanathkr/go-yaml@v0.0.0 incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -------------------------------------------------------------------------------- -Dependency : github.com/sanathkr/yaml -Version: v1.0.1-0.20170819201035-0056894fa522 +Dependency : github.com/spf13/cast +Version: v1.3.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/sanathkr/yaml@v1.0.1-0.20170819201035-0056894fa522/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/spf13/cast@v1.3.0/LICENSE: The MIT License (MIT) -Copyright (c) 2014 Sam Ghods +Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -35204,143 +37885,17 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - --------------------------------------------------------------------------------- -Dependency : github.com/santhosh-tekuri/jsonschema -Version: v1.2.4 -Licence type (autodetected): BSD-3-Clause --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/santhosh-tekuri/jsonschema@v1.2.4/LICENSE: - -Copyright (c) 2017 Santhosh Kumar Tekuri. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -------------------------------------------------------------------------------- -Dependency : github.com/satori/go.uuid -Version: v1.2.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/satori/go.uuid@v1.2.0/LICENSE: - -Copyright (C) 2013-2018 by Maxim Bublis - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/sergi/go-diff -Version: v1.1.0 -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/sergi/go-diff@v1.1.0/LICENSE: - -Copyright (c) 2012-2016 The go-diff Authors. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a -copy of this software and associated documentation files (the "Software"), -to deal in the Software without restriction, including without limitation -the rights to use, copy, modify, merge, publish, distribute, sublicense, -and/or sell copies of the Software, and to permit persons to whom the -Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. - - - --------------------------------------------------------------------------------- -Dependency : github.com/sirupsen/logrus -Version: v1.4.2 +Dependency : github.com/spf13/jwalterweatherman +Version: v1.0.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/sirupsen/logrus@v1.4.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/spf13/jwalterweatherman@v1.0.0/LICENSE: The MIT License (MIT) -Copyright (c) 2014 Simon Eskildsen +Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -35349,60 +37904,28 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - - --------------------------------------------------------------------------------- -Dependency : github.com/smartystreets/assertions -Version: v0.0.0-20180927180507-b2de0cb4f26d -Licence type (autodetected): MIT --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/smartystreets/assertions@v0.0.0-20180927180507-b2de0cb4f26d/LICENSE.md: - -Copyright (c) 2016 SmartyStreets, LLC - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -NOTE: Various optional and subordinate components carry their own licensing -requirements and restrictions. Use of those components is subject to the terms -and conditions outlined the respective license of each component. - - -------------------------------------------------------------------------------- -Dependency : github.com/smartystreets/goconvey -Version: v0.0.0-20190330032615-68dc04aab96a +Dependency : github.com/spf13/viper +Version: v1.3.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/smartystreets/goconvey@v0.0.0-20190330032615-68dc04aab96a/LICENSE.md: +Contents of probable licence file $GOMODCACHE/github.com/spf13/viper@v1.3.2/LICENSE: -Copyright (c) 2016 SmartyStreets, LLC +The MIT License (MIT) + +Copyright (c) 2014 Steve Francia Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -35422,195 +37945,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -NOTE: Various optional and subordinate components carry their own licensing -requirements and restrictions. Use of those components is subject to the terms -and conditions outlined the respective license of each component. - - --------------------------------------------------------------------------------- -Dependency : github.com/spf13/afero -Version: v1.2.2 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/spf13/afero@v1.2.2/LICENSE.txt: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx Version: v0.2.0 @@ -35677,6 +38011,38 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Dependency : github.com/ugorji/go +Version: v1.1.8 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/ugorji/go@v1.1.8/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2012-2015 Ugorji Nwoke. +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/urfave/cli Version: v0.0.0-20171014202726-7bc6a0acffa5 @@ -37420,6 +39786,25 @@ Contents of probable licence file $GOMODCACHE/github.com/xeipuuv/gojsonschema@v0 limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/xordataexchange/crypt +Version: v0.0.3-0.20170626215501-b2862e3d0a77 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/xordataexchange/crypt@v0.0.3-0.20170626215501-b2862e3d0a77/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2014 XOR Data Exchange, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/yuin/gopher-lua Version: v0.0.0-20170403160031-b402f3114ec7 diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index cd08240c9a3..7eed06a497c 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -47,3 +47,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test auditbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/auditbeat/cmd/root.go b/auditbeat/cmd/root.go index 89d0bfd20ca..bf3167e7106 100644 --- a/auditbeat/cmd/root.go +++ b/auditbeat/cmd/root.go @@ -35,7 +35,7 @@ const ( Name = "auditbeat" // ecsVersion specifies the version of ECS that Auditbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // RootCmd for running auditbeat. diff --git a/auditbeat/docs/fields.asciidoc b/auditbeat/docs/fields.asciidoc index 7ba194357ee..cb95eb4da12 100644 --- a/auditbeat/docs/fields.asciidoc +++ b/auditbeat/docs/fields.asciidoc @@ -2914,8 +2914,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + diff --git a/auditbeat/module/file_integrity/event.go b/auditbeat/module/file_integrity/event.go index 41e4d5a3795..1ee28b7ce35 100644 --- a/auditbeat/module/file_integrity/event.go +++ b/auditbeat/module/file_integrity/event.go @@ -257,7 +257,7 @@ func buildMetricbeatEvent(e *Event, existedBefore bool) mb.Event { if e.Info.Type == FileType { if extension := filepath.Ext(e.Path); extension != "" { - file["extension"] = extension + file["extension"] = strings.TrimLeft(extension, ".") } if mimeType := getMimeType(e.Path); mimeType != "" { file["mime_type"] = mimeType diff --git a/auditbeat/module/file_integrity/event_test.go b/auditbeat/module/file_integrity/event_test.go index 79d1309903d..efaafd02041 100644 --- a/auditbeat/module/file_integrity/event_test.go +++ b/auditbeat/module/file_integrity/event_test.go @@ -28,6 +28,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" ) @@ -295,7 +296,11 @@ func TestBuildEvent(t *testing.T) { assertHasKey(t, fields, "event.type") assertHasKey(t, fields, "file.path") - assertHasKey(t, fields, "file.extension") + if assertHasKey(t, fields, "file.extension") { + ext, err := fields.GetValue("file.extension") + require.NoError(t, err) + assert.Equal(t, ext, "txt") + } assertHasKey(t, fields, "file.target_path") assertHasKey(t, fields, "file.inode") assertHasKey(t, fields, "file.uid") @@ -427,10 +432,12 @@ func mustDecodeHex(v string) []byte { return data } -func assertHasKey(t testing.TB, m common.MapStr, key string) { +func assertHasKey(t testing.TB, m common.MapStr, key string) bool { t.Helper() found, err := m.HasKey(key) if err != nil || !found { t.Errorf("key %v not found: %v", key, err) + return false } + return true } diff --git a/deploy/cloudfoundry/filebeat/filebeat.yml b/deploy/cloudfoundry/filebeat/filebeat.yml index f4c4943d4bb..f3d58f079a1 100644 --- a/deploy/cloudfoundry/filebeat/filebeat.yml +++ b/deploy/cloudfoundry/filebeat/filebeat.yml @@ -11,7 +11,7 @@ filebeat.inputs: #doppler_address: ${DOPPLER_ADDRESS} #uaa_address: ${UAA_ADDRESS} #rlp_address: ${RLP_ADDRESS} - #shard_id: ${SHARD_ID} + shard_id: ${SHARD_ID} #version: v1 diff --git a/deploy/cloudfoundry/filebeat/manifest.yml b/deploy/cloudfoundry/filebeat/manifest.yml index 26ec5a0b958..5c424823cb3 100644 --- a/deploy/cloudfoundry/filebeat/manifest.yml +++ b/deploy/cloudfoundry/filebeat/manifest.yml @@ -1,6 +1,6 @@ applications: - name: filebeat - memory: 256M + memory: 512M instances: 1 buildpacks: - binary_buildpack diff --git a/deploy/cloudfoundry/metricbeat/manifest.yml b/deploy/cloudfoundry/metricbeat/manifest.yml index 40f0459b8da..1a2bb025683 100644 --- a/deploy/cloudfoundry/metricbeat/manifest.yml +++ b/deploy/cloudfoundry/metricbeat/manifest.yml @@ -1,6 +1,6 @@ applications: - name: metricbeat - memory: 256M + memory: 512M instances: 1 buildpacks: - binary_buildpack diff --git a/deploy/cloudfoundry/metricbeat/metricbeat.yml b/deploy/cloudfoundry/metricbeat/metricbeat.yml index c89c00ce983..12cfee75f00 100644 --- a/deploy/cloudfoundry/metricbeat/metricbeat.yml +++ b/deploy/cloudfoundry/metricbeat/metricbeat.yml @@ -11,7 +11,8 @@ metricbeat.modules: #doppler_address: ${DOPPLER_ADDRESS} #uaa_address: ${UAA_ADDRESS} #rlp_address: ${RLP_ADDRESS} - #shard_id: ${SHARD_ID} + shard_id: ${SHARD_ID} + #version: v1 #================================ Outputs ===================================== diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go index 2ed09db711e..8b366bad315 100644 --- a/dev-tools/mage/integtest_docker.go +++ b/dev-tools/mage/integtest_docker.go @@ -27,6 +27,7 @@ import ( "runtime" "strings" "sync" + "time" "github.com/pkg/errors" @@ -246,5 +247,17 @@ func dockerComposeBuildImages() error { os.Stderr, "docker-compose", args..., ) + + // This sleep is to avoid hitting the docker build issues when resources are not available. + if err != nil { + fmt.Println(">> Building docker images again") + time.Sleep(10) + _, err = sh.Exec( + composeEnv, + out, + os.Stderr, + "docker-compose", args..., + ) + } return err } diff --git a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl index 91f043d2799..d4c8d6e4645 100644 --- a/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl +++ b/dev-tools/packaging/templates/docker/docker-entrypoint.elastic-agent.tmpl @@ -13,8 +13,8 @@ set -eo pipefail # KIBANA_USERNAME - username for accessing kibana API [elastic] function setup(){ - curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} - curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/setup \ + curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/fleet/setup -H 'kbn-xsrf: true' -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} + curl -X POST ${KIBANA_HOST:-http://localhost:5601}/api/fleet/agents/setup \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: true' \ -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} @@ -27,7 +27,7 @@ function enroll(){ if [[ -n "${FLEET_ENROLLMENT_TOKEN}" ]]; then apikey="${FLEET_ENROLLMENT_TOKEN}" else - enrollResp=$(curl ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys \ + enrollResp=$(curl ${KIBANA_HOST:-http://localhost:5601}/api/fleet/enrollment-api-keys \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: true' \ -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} ) @@ -40,7 +40,7 @@ function enroll(){ local apikeyId=$(echo $enrollResp | jq -r '.list[0].id') echo $apikeyId - enrollResp=$(curl ${KIBANA_HOST:-http://localhost:5601}/api/ingest_manager/fleet/enrollment-api-keys/$apikeyId \ + enrollResp=$(curl ${KIBANA_HOST:-http://localhost:5601}/api/fleet/enrollment-api-keys/$apikeyId \ -H 'Content-Type: application/json' \ -H 'kbn-xsrf: true' \ -u ${KIBANA_USERNAME:-elastic}:${KIBANA_PASSWORD:-changeme} ) diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 28e4e4a7bc1..09dbe948c72 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -46,3 +46,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test filebeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index c920b7dbec8..7eceb559f16 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -11,6 +11,7 @@ filebeat.inputs: # # Possible options are: # * log: Reads every line of the log file (default) +# * filestream: Improved version of log input. Experimental. # * stdin: Reads the standard in #------------------------------ Log input -------------------------------- @@ -231,6 +232,145 @@ filebeat.inputs: # Defines if inputs is enabled #enabled: true +#--------------------------- Filestream input ---------------------------- +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + # To fetch all ".log" files from a specific level of subdirectories + # /var/log/*/*.log can be used. + # For each file found under this path, a harvester is started. + # Make sure not file is defined twice as this can lead to unexpected behaviour. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Configure the file encoding for reading files with international characters + # following the W3C recommendation for HTML5 (http://www.w3.org/TR/encoding). + # Some sample encodings: + # plain, utf-8, utf-16be-bom, utf-16be, utf-16le, big5, gb18030, gbk, + # hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ... + #encoding: plain + + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, no lines are dropped. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, all the lines are exported. + #include_lines: ['^ERR', '^WARN'] + + ### Prospector options + + # How often the input checks for new files in the paths that are specified + # for harvesting. Specify 1s to scan the directory as frequently as possible + # without causing Filebeat to scan too frequently. Default: 10s. + #prospector.scanner.check_interval: 10s + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Expand "**" patterns into regular glob patterns. + #prospector.scanner.recursive_glob: true + + # If symlinks is enabled, symlinks are opened and harvested. The harvester is opening the + # original for harvesting but will report the symlink name as source. + #prospector.scanner.symlinks: false + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + #clean_inactive: 0 + + # Removes the state for file which cannot be found on disk anymore immediately + #clean_removed: true + + # Method to determine if two files are the same or not. By default + # the Beat considers two files the same if their inode and device id are the same. + #file_identity.native: ~ + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + # Set to true to publish fields with null values in events. + #keep_null: false + + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + + # Ignore files which were modified more then the defined timespan in the past. + # ignore_older is disabled by default, so no files are ignored by setting it to 0. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #ignore_older: 0 + + # Defines the buffer size every harvester uses when fetching the file + #harvester_buffer_size: 16384 + + # Maximum number of bytes a single log event can have + # All bytes after max_bytes are discarded and not sent. The default is 10MB. + # This is especially useful for multiline log messages which can get large. + #message_max_bytes: 10485760 + + # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + #line_terminator: auto + + # The Ingest Node pipeline ID associated with this input. If this is set, it + # overwrites the pipeline option from the Elasticsearch output. + #pipeline: + + # Backoff values define how aggressively filebeat crawls new files for updates + # The default values can be used in most cases. Backoff defines how long it is waited + # to check a file again after EOF is reached. Default is 1s which means the file + # is checked every second if new lines were added. This leads to a near real time crawling. + # Every time a new line appears, backoff is reset to the initial value. + #backoff.init: 1s + + # Max backoff defines what the maximum backoff time is. After having backed off multiple times + # from checking the files, the waiting time will never exceed max_backoff independent of the + # backoff factor. Having it set to 10s means in the worst case a new line can be added to a log + # file after having backed off multiple times, it takes a maximum of 10s to read the new line + #backoff.max: 10s + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #close.on_state_change.inactive: 5m + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.on_state_change.renamed: false + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + #close.on_state_change.removed: true + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.eof: false + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.after_interval: 0 + #----------------------------- Stdin input ------------------------------- # Configuration to use stdin input #- type: stdin diff --git a/filebeat/_meta/config/filebeat.inputs.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.yml.tmpl index a7bd1b5eaa6..70d52cbb9c6 100644 --- a/filebeat/_meta/config/filebeat.inputs.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.yml.tmpl @@ -49,3 +49,32 @@ filebeat.inputs: # that was (not) matched before or after or as long as a pattern is not matched based on negate. # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after + +# filestream is an experimental input. It is going to replace log input in the future. +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 8a145ff8724..b4f6a158ad7 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1884,6 +1884,46 @@ type: keyword The error reason if the executed action failed. +type: keyword + +-- + +*`aws.elb.target_port`*:: ++ +-- +List of IP addresses and ports for the targets that processed this request. + + +type: keyword + +-- + +*`aws.elb.target_status_code`*:: ++ +-- +List of status codes from the responses of the targets. + + +type: keyword + +-- + +*`aws.elb.classification`*:: ++ +-- +The classification for desync mitigation. + + +type: keyword + +-- + +*`aws.elb.classification_reason`*:: ++ +-- +The classification reason code. + + type: keyword -- @@ -26581,6 +26621,153 @@ type: keyword -- This key captures values or decorators used within a registry entry +type: keyword + +-- + +[float] +=== cisco.umbrella + +Fields for Cisco Umbrella. + + + +*`cisco.umbrella.identities`*:: ++ +-- +An array of the different identities related to the event. + + +type: keyword + +-- + +*`cisco.umbrella.categories`*:: ++ +-- +The security or content categories that the destination matches. + + +type: keyword + +-- + +*`cisco.umbrella.policy_identity_type`*:: ++ +-- +The first identity type matched with this request. Available in version 3 and above. + + +type: keyword + +-- + +*`cisco.umbrella.identity_types`*:: ++ +-- +The type of identity that made the request. For example, Roaming Computer or Network. + + +type: keyword + +-- + +*`cisco.umbrella.blocked_categories`*:: ++ +-- +The categories that resulted in the destination being blocked. Available in version 4 and above. + + +type: keyword + +-- + +*`cisco.umbrella.content_type`*:: ++ +-- +The type of web content, typically text/html. + + +type: keyword + +-- + +*`cisco.umbrella.sha_sha256`*:: ++ +-- +Hex digest of the response content. + + +type: keyword + +-- + +*`cisco.umbrella.av_detections`*:: ++ +-- +The detection name according to the antivirus engine used in file inspection. + + +type: keyword + +-- + +*`cisco.umbrella.puas`*:: ++ +-- +A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. + + +type: keyword + +-- + +*`cisco.umbrella.amp_disposition`*:: ++ +-- +The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. + + +type: keyword + +-- + +*`cisco.umbrella.amp_malware_name`*:: ++ +-- +If Malicious, the name of the malware according to AMP. + + +type: keyword + +-- + +*`cisco.umbrella.amp_score`*:: ++ +-- +The score of the malware from AMP. This field is not currently used and will be blank. + + +type: keyword + +-- + +*`cisco.umbrella.datacenter`*:: ++ +-- +The name of the Umbrella Data Center that processed the user-generated traffic. + + +type: keyword + +-- + +*`cisco.umbrella.origin_id`*:: ++ +-- +The unique identity of the network tunnel. + + type: keyword -- @@ -44041,8 +44228,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + @@ -87929,6 +88123,973 @@ type: keyword -- This key captures values or decorators used within a registry entry +type: keyword + +-- + +[float] +=== juniper.srx + +Module for parsing junipersrx syslog. + + + +*`juniper.srx.reason`*:: ++ +-- +reason + + +type: keyword + +-- + +*`juniper.srx.connection_tag`*:: ++ +-- +connection tag + + +type: keyword + +-- + +*`juniper.srx.service_name`*:: ++ +-- +service name + + +type: keyword + +-- + +*`juniper.srx.nat_connection_tag`*:: ++ +-- +nat connection tag + + +type: keyword + +-- + +*`juniper.srx.src_nat_rule_type`*:: ++ +-- +src nat rule type + + +type: keyword + +-- + +*`juniper.srx.src_nat_rule_name`*:: ++ +-- +src nat rule name + + +type: keyword + +-- + +*`juniper.srx.dst_nat_rule_type`*:: ++ +-- +dst nat rule type + + +type: keyword + +-- + +*`juniper.srx.dst_nat_rule_name`*:: ++ +-- +dst nat rule name + + +type: keyword + +-- + +*`juniper.srx.protocol_id`*:: ++ +-- +protocol id + + +type: keyword + +-- + +*`juniper.srx.policy_name`*:: ++ +-- +policy name + + +type: keyword + +-- + +*`juniper.srx.session_id_32`*:: ++ +-- +session id 32 + + +type: keyword + +-- + +*`juniper.srx.session_id`*:: ++ +-- +session id + + +type: keyword + +-- + +*`juniper.srx.outbound_packets`*:: ++ +-- +packets from client + + +type: integer + +-- + +*`juniper.srx.outbound_bytes`*:: ++ +-- +bytes from client + + +type: integer + +-- + +*`juniper.srx.inbound_packets`*:: ++ +-- +packets from server + + +type: integer + +-- + +*`juniper.srx.inbound_bytes`*:: ++ +-- +bytes from server + + +type: integer + +-- + +*`juniper.srx.elapsed_time`*:: ++ +-- +elapsed time + + +type: date + +-- + +*`juniper.srx.application`*:: ++ +-- +application + + +type: keyword + +-- + +*`juniper.srx.nested_application`*:: ++ +-- +nested application + + +type: keyword + +-- + +*`juniper.srx.username`*:: ++ +-- +username + + +type: keyword + +-- + +*`juniper.srx.roles`*:: ++ +-- +roles + + +type: keyword + +-- + +*`juniper.srx.encrypted`*:: ++ +-- +encrypted + + +type: keyword + +-- + +*`juniper.srx.application_category`*:: ++ +-- +application category + + +type: keyword + +-- + +*`juniper.srx.application_sub_category`*:: ++ +-- +application sub category + + +type: keyword + +-- + +*`juniper.srx.application_characteristics`*:: ++ +-- +application characteristics + + +type: keyword + +-- + +*`juniper.srx.secure_web_proxy_session_type`*:: ++ +-- +secure web proxy session type + + +type: keyword + +-- + +*`juniper.srx.peer_session_id`*:: ++ +-- +peer session id + + +type: keyword + +-- + +*`juniper.srx.peer_source_address`*:: ++ +-- +peer source address + + +type: ip + +-- + +*`juniper.srx.peer_source_port`*:: ++ +-- +peer source port + + +type: integer + +-- + +*`juniper.srx.peer_destination_address`*:: ++ +-- +peer destination address + + +type: ip + +-- + +*`juniper.srx.peer_destination_port`*:: ++ +-- +peer destination port + + +type: integer + +-- + +*`juniper.srx.hostname`*:: ++ +-- +hostname + + +type: keyword + +-- + +*`juniper.srx.src_vrf_grp`*:: ++ +-- +src_vrf_grp + + +type: keyword + +-- + +*`juniper.srx.dst_vrf_grp`*:: ++ +-- +dst_vrf_grp + + +type: keyword + +-- + +*`juniper.srx.icmp_type`*:: ++ +-- +icmp type + + +type: integer + +-- + +*`juniper.srx.process`*:: ++ +-- +process that generated the message + + +type: keyword + +-- + +*`juniper.srx.apbr_rule_type`*:: ++ +-- +apbr rule type + + +type: keyword + +-- + +*`juniper.srx.dscp_value`*:: ++ +-- +apbr rule type + + +type: integer + +-- + +*`juniper.srx.logical_system_name`*:: ++ +-- +logical system name + + +type: keyword + +-- + +*`juniper.srx.profile_name`*:: ++ +-- +profile name + + +type: keyword + +-- + +*`juniper.srx.routing_instance`*:: ++ +-- +routing instance + + +type: keyword + +-- + +*`juniper.srx.rule_name`*:: ++ +-- +rule name + + +type: keyword + +-- + +*`juniper.srx.uplink_tx_bytes`*:: ++ +-- +uplink tx bytes + + +type: integer + +-- + +*`juniper.srx.uplink_rx_bytes`*:: ++ +-- +uplink rx bytes + + +type: integer + +-- + +*`juniper.srx.obj`*:: ++ +-- +url path + + +type: keyword + +-- + +*`juniper.srx.url`*:: ++ +-- +url domain + + +type: keyword + +-- + +*`juniper.srx.profile`*:: ++ +-- +filter profile + + +type: keyword + +-- + +*`juniper.srx.category`*:: ++ +-- +filter category + + +type: keyword + +-- + +*`juniper.srx.filename`*:: ++ +-- +filename + + +type: keyword + +-- + +*`juniper.srx.temporary_filename`*:: ++ +-- +temporary_filename + + +type: keyword + +-- + +*`juniper.srx.name`*:: ++ +-- +name + + +type: keyword + +-- + +*`juniper.srx.error_message`*:: ++ +-- +error_message + + +type: keyword + +-- + +*`juniper.srx.error_code`*:: ++ +-- +error_code + + +type: keyword + +-- + +*`juniper.srx.action`*:: ++ +-- +action + + +type: keyword + +-- + +*`juniper.srx.protocol`*:: ++ +-- +protocol + + +type: keyword + +-- + +*`juniper.srx.protocol_name`*:: ++ +-- +protocol name + + +type: keyword + +-- + +*`juniper.srx.type`*:: ++ +-- +type + + +type: keyword + +-- + +*`juniper.srx.repeat_count`*:: ++ +-- +repeat count + + +type: integer + +-- + +*`juniper.srx.alert`*:: ++ +-- +repeat alert + + +type: keyword + +-- + +*`juniper.srx.message_type`*:: ++ +-- +message type + + +type: keyword + +-- + +*`juniper.srx.threat_severity`*:: ++ +-- +threat severity + + +type: keyword + +-- + +*`juniper.srx.application_name`*:: ++ +-- +application name + + +type: keyword + +-- + +*`juniper.srx.attack_name`*:: ++ +-- +attack name + + +type: keyword + +-- + +*`juniper.srx.index`*:: ++ +-- +index + + +type: keyword + +-- + +*`juniper.srx.message`*:: ++ +-- +mesagge + + +type: keyword + +-- + +*`juniper.srx.epoch_time`*:: ++ +-- +epoch time + + +type: date + +-- + +*`juniper.srx.packet_log_id`*:: ++ +-- +packet log id + + +type: integer + +-- + +*`juniper.srx.export_id`*:: ++ +-- +packet log id + + +type: integer + +-- + +*`juniper.srx.ddos_application_name`*:: ++ +-- +ddos application name + + +type: keyword + +-- + +*`juniper.srx.connection_hit_rate`*:: ++ +-- +connection hit rate + + +type: integer + +-- + +*`juniper.srx.time_scope`*:: ++ +-- +time scope + + +type: keyword + +-- + +*`juniper.srx.context_hit_rate`*:: ++ +-- +context hit rate + + +type: integer + +-- + +*`juniper.srx.context_value_hit_rate`*:: ++ +-- +context value hit rate + + +type: integer + +-- + +*`juniper.srx.time_count`*:: ++ +-- +time count + + +type: integer + +-- + +*`juniper.srx.time_period`*:: ++ +-- +time period + + +type: integer + +-- + +*`juniper.srx.context_value`*:: ++ +-- +context value + + +type: keyword + +-- + +*`juniper.srx.context_name`*:: ++ +-- +context name + + +type: keyword + +-- + +*`juniper.srx.ruleebase_name`*:: ++ +-- +ruleebase name + + +type: keyword + +-- + +*`juniper.srx.verdict_source`*:: ++ +-- +verdict source + + +type: keyword + +-- + +*`juniper.srx.verdict_number`*:: ++ +-- +verdict number + + +type: integer + +-- + +*`juniper.srx.file_category`*:: ++ +-- +file category + + +type: keyword + +-- + +*`juniper.srx.sample_sha256`*:: ++ +-- +sample sha256 + + +type: keyword + +-- + +*`juniper.srx.malware_info`*:: ++ +-- +malware info + + +type: keyword + +-- + +*`juniper.srx.client_ip`*:: ++ +-- +client ip + + +type: ip + +-- + +*`juniper.srx.tenant_id`*:: ++ +-- +tenant id + + +type: keyword + +-- + +*`juniper.srx.timestamp`*:: ++ +-- +timestamp + + +type: date + +-- + +*`juniper.srx.th`*:: ++ +-- +th + + +type: keyword + +-- + +*`juniper.srx.status`*:: ++ +-- +status + + +type: keyword + +-- + +*`juniper.srx.state`*:: ++ +-- +state + + +type: keyword + +-- + +*`juniper.srx.file_hash_lookup`*:: ++ +-- +file hash lookup + + +type: keyword + +-- + +*`juniper.srx.file_name`*:: ++ +-- +file name + + +type: keyword + +-- + +*`juniper.srx.action_detail`*:: ++ +-- +action detail + + +type: keyword + +-- + +*`juniper.srx.sub_category`*:: ++ +-- +sub category + + +type: keyword + +-- + +*`juniper.srx.feed_name`*:: ++ +-- +feed name + + +type: keyword + +-- + +*`juniper.srx.occur_count`*:: ++ +-- +occur count + + +type: integer + +-- + +*`juniper.srx.tag`*:: ++ +-- +system log message tag, which uniquely identifies the message. + + type: keyword -- @@ -94409,6 +95570,433 @@ type: keyword -- +[float] +=== microsoft.m365_defender + +Module for ingesting Microsoft Defender ATP. + + + +*`microsoft.m365_defender.incidentId`*:: ++ +-- +Unique identifier to represent the incident. + + +type: keyword + +-- + +*`microsoft.m365_defender.redirectIncidentId`*:: ++ +-- +Only populated in case an incident is being grouped together with another incident, as part of the incident processing logic. + + +type: keyword + +-- + +*`microsoft.m365_defender.incidentName`*:: ++ +-- +Name of the Incident. + + +type: keyword + +-- + +*`microsoft.m365_defender.determination`*:: ++ +-- +Specifies the determination of the incident. The property values are: NotAvailable, Apt, Malware, SecurityPersonnel, SecurityTesting, UnwantedSoftware, Other. + + +type: keyword + +-- + +*`microsoft.m365_defender.investigationState`*:: ++ +-- +The current state of the Investigation. + + +type: keyword + +-- + +*`microsoft.m365_defender.assignedTo`*:: ++ +-- +Owner of the alert. + + +type: keyword + +-- + +*`microsoft.m365_defender.tags`*:: ++ +-- +Array of custom tags associated with an incident, for example to flag a group of incidents with a common characteristic. + + +type: keyword + +-- + +*`microsoft.m365_defender.status`*:: ++ +-- +Specifies the current status of the alert. Possible values are: 'Unknown', 'New', 'InProgress' and 'Resolved'. + + +type: keyword + +-- + +*`microsoft.m365_defender.classification`*:: ++ +-- +Specification of the alert. Possible values are: 'Unknown', 'FalsePositive', 'TruePositive'. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.incidentId`*:: ++ +-- +Unique identifier to represent the incident this alert is associated with. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.resolvedTime`*:: ++ +-- +Time when alert was resolved. + + +type: date + +-- + +*`microsoft.m365_defender.alerts.status`*:: ++ +-- +Categorize alerts (as New, Active, or Resolved). + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.severity`*:: ++ +-- +The severity of the related alert. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.creationTime`*:: ++ +-- +Time when alert was first created. + + +type: date + +-- + +*`microsoft.m365_defender.alerts.lastUpdatedTime`*:: ++ +-- +Time when alert was last updated. + + +type: date + +-- + +*`microsoft.m365_defender.alerts.investigationId`*:: ++ +-- +The automated investigation id triggered by this alert. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.userSid`*:: ++ +-- +The SID of the related user + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.detectionSource`*:: ++ +-- +The service that initially detected the threat. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.classification`*:: ++ +-- +The specification for the incident. The property values are: Unknown, FalsePositive, TruePositive or null. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.investigationState`*:: ++ +-- +Information on the investigation's current status. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.determination`*:: ++ +-- +Specifies the determination of the incident. The property values are: NotAvailable, Apt, Malware, SecurityPersonnel, SecurityTesting, UnwantedSoftware, Other or null + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.assignedTo`*:: ++ +-- +Owner of the incident, or null if no owner is assigned. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.actorName`*:: ++ +-- +The activity group, if any, the associated with this alert. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.threatFamilyName`*:: ++ +-- +Threat family associated with this alert. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.mitreTechniques`*:: ++ +-- +The attack techniques, as aligned with the MITRE ATT&CK™ framework. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.entityType`*:: ++ +-- +Entities that have been identified to be part of, or related to, a given alert. The properties values are: User, Ip, Url, File, Process, MailBox, MailMessage, MailCluster, Registry. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.accountName`*:: ++ +-- +Account name of the related user. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.mailboxDisplayName`*:: ++ +-- +The display name of the related mailbox. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.mailboxAddress`*:: ++ +-- +The mail address of the related mailbox. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.clusterBy`*:: ++ +-- +A list of metadata if the entityType is MailCluster. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.sender`*:: ++ +-- +The sender for the related email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.recipient`*:: ++ +-- +The recipient for the related email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.subject`*:: ++ +-- +The subject for the related email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.deliveryAction`*:: ++ +-- +The delivery status for the related email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.securityGroupId`*:: ++ +-- +The Security Group ID for the user related to the email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.securityGroupName`*:: ++ +-- +The Security Group Name for the user related to the email message. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.registryHive`*:: ++ +-- +Reference to which Hive in registry the event is related to, if eventType is registry. Example: HKEY_LOCAL_MACHINE. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.registryKey`*:: ++ +-- +Reference to the related registry key to the event. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.registryValueType`*:: ++ +-- +Value type of the registry key/value pair related to the event. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.deviceId`*:: ++ +-- +The unique ID of the device related to the event. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.entities.ipAddress`*:: ++ +-- +The related IP address to the event. + + +type: keyword + +-- + +*`microsoft.m365_defender.alerts.devices`*:: ++ +-- +The devices related to the investigation. + + +type: flattened + +-- + [[exported-fields-misp]] == MISP fields diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index c12f818caca..d087826245e 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -10,13 +10,15 @@ This file is generated! See scripts/docs_collector.py == Cisco module -This is a module for Cisco network device's logs. It includes the following +This is a module for Cisco network device's logs and Cisco Umbrella. It includes the following filesets for receiving logs over syslog or read from a file: - `asa` fileset: supports Cisco ASA firewall logs. - `ftd` fileset: supports Cisco Firepower Threat Defense logs. - `ios` fileset: supports Cisco IOS router and switch logs. - `nexus` fileset: supports Cisco Nexus switch logs. +- `meraki` fileset: supports Cisco Meraki logs. +- `umbrella` fileset: supports Cisco Umbrella logs. Cisco ASA devices also support exporting flow records using NetFlow, which is supported by the {filebeat-ref}/filebeat-module-netflow.html[netflow module] in @@ -32,6 +34,8 @@ The module is by default configured to run via syslog on port 9001 for ASA and port 9002 for IOS. However it can also be configured to read from a file path. See the following example. +Cisco Umbrella publishes its logs in a compressed CSV format to a S3 bucket. + ["source","yaml",subs="attributes"] ----- - module: cisco @@ -379,6 +383,59 @@ will be found under `rsa.raw`. The default is false. :fileset_ex!: +[float] +==== `umbrella` fileset settings + +The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. + +To configure Cisco Umbrella to log to either your own S3 bucket or one that is managed by Cisco please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide.] + +This fileset supports all 4 log types: +- Proxy +- Cloud Firewall +- IP Logs +- DNS logs + +The Cisco Umbrella fileset depends on the original file path structure being followed. This structure is documented https://docs.umbrella.com/deployment-umbrella/docs/log-formats-and-versioning[Umbrella Log Formats and Versioning]: + +/--
/--
---.csv.gz +dnslogs/--/----.csv.gz + +When configuring the fileset, please ensure that the Queue URL is set to the root folder that includes each of these subfolders above. + +Example config: + +[source,yaml] +---- +- module: cisco + umbrella: + enabled: true + var.input: s3 + var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + var.access_key_id: 123456 + var.secret_access_key: PASSWORD +---- + +*`var.input`*:: + +The input from which messages are read. Can be S3 or file. + +*`var.queue_url`*:: + +The URL to the SQS queue if the input type is S3. + +*`var.access_key_id`*:: + +The ID for the access key used to read from the SQS queue. + +*`var.secret_access_key`*:: + +The secret token used for authenticating to the SQS queue. + +:has-dashboards!: + +:fileset_ex!: + [float] === Example dashboard diff --git a/filebeat/docs/modules/juniper.asciidoc b/filebeat/docs/modules/juniper.asciidoc index 047e847bc5a..a2d2a0100d3 100644 --- a/filebeat/docs/modules/juniper.asciidoc +++ b/filebeat/docs/modules/juniper.asciidoc @@ -10,18 +10,131 @@ This file is generated! See scripts/docs_collector.py == Juniper module -experimental[] +This is a module for ingesting data from the different Juniper Products. Currently supports these filesets: -This is a module for receiving Juniper JUNOS logs over Syslog or a file. +- `srx` fileset: Supports Juniper SRX logs +- `junos` fileset: Supports Juniper JUNOS logs +- `netscreen` fileset: Supports Juniper Netscreen logs include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] -:fileset_ex: junos - include::../include/config-option-intro.asciidoc[] +:fileset_ex: srx +beta[] + +[float] +==== `srx` fileset settings + +The Juniper-SRX module only supports syslog messages in the format "structured-data + brief" https://www.juniper.net/documentation/en_US/junos/topics/reference/configuration-statement/structured-data-edit-system.html[JunOS Documentation structured-data] + +To configure a remote syslog destination, please reference the https://kb.juniper.net/InfoCenter/index?page=content&id=kb16502[SRX Getting Started - Configure System Logging]. + +The following processes and tags are supported: + +[options="header"] +|============================================================== +| JunOS processes | JunOS tags | +| RT_FLOW | RT_FLOW_SESSION_CREATE | +| | RT_FLOW_SESSION_CLOSE | +| | RT_FLOW_SESSION_DENY | +| | APPTRACK_SESSION_CREATE | +| | APPTRACK_SESSION_CLOSE | +| | APPTRACK_SESSION_VOL_UPDATE | +| RT_IDS | RT_SCREEN_TCP | +| | RT_SCREEN_UDP | +| | RT_SCREEN_ICMP | +| | RT_SCREEN_IP | +| | RT_SCREEN_TCP_DST_IP | +| | RT_SCREEN_TCP_SRC_IP | +| RT_UTM | WEBFILTER_URL_PERMITTED | +| | WEBFILTER_URL_BLOCKED | +| | AV_VIRUS_DETECTED_MT | +| | CONTENT_FILTERING_BLOCKED_MT | +| | ANTISPAM_SPAM_DETECTED_MT | +| RT_IDP | IDP_ATTACK_LOG_EVENT | +| | IDP_APPDDOS_APP_STATE_EVENT | +| RT_AAMW | SRX_AAMW_ACTION_LOG | +| | AAMW_MALWARE_EVENT_LOG | +| | AAMW_HOST_INFECTED_EVENT_LOG | +| | AAMW_ACTION_LOG | +| RT_SECINTEL | SECINTEL_ACTION_LOG | +|============================================================== + +The syslog format choosen should be `Default`. + +[float] +=== Compatibility + +This module has been tested against JunOS version 19.x and 20.x. +Versions above this are expected to work but have not been tested. + +[source,yaml] +---- +- module: sophosxg + firewall: + enabled: true + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9006 +---- + +include::../include/var-paths.asciidoc[] + +*`var.input`*:: + +The input to use, can be either the value `tcp`, `udp` or `file`. + +*`var.syslog_host`*:: + +The interface to listen to all syslog traffic. Defaults to localhost. +Set to 0.0.0.0 to bind to all available interfaces. + +*`var.syslog_port`*:: + +The port to listen for syslog traffic. Defaults to 9006. + + +[float] +==== Juniper SRX ECS fields + +This is a list of JunOS fields that are mapped to ECS. + +[options="header"] +|============================================================== +| Juniper SRX Fields | ECS Fields | +| application-risk | event.risk_score | +| bytes-from-client | source.bytes | +| bytes-from-server | destination.bytes | +| destination-interface-name | observer.egress.interface.name | +| destination-zone-name | observer.egress.zone | +| destination-address | destination.ip | +| destination-port | destination.port | +| dst_domainname | url.domain | +| elapsed-time | event.duration | +| filename | file.name | +| nat-destination-address | destination.nat.ip | +| nat-destination-port | destination.nat.port | +| nat-source-address | source.nat.ip | +| nat-source-port | source.nat.port | +| message | message | +| obj | url.path | +| packets-from-client | source.packets | +| packets-from-server | destination.packets | +| policy-name | rule.name | +| protocol | network.transport | +| source-address | source.ip | +| source-interface-name | observer.ingress.interface.name| +| source-port | source.port | +| source-zone-name | observer.ingress.zone | +| url | url.domain | +|============================================================== + + +:fileset_ex: junos + [float] ==== `junos` fileset settings diff --git a/filebeat/docs/modules/microsoft.asciidoc b/filebeat/docs/modules/microsoft.asciidoc index 513ca155be6..5edbbf027d0 100644 --- a/filebeat/docs/modules/microsoft.asciidoc +++ b/filebeat/docs/modules/microsoft.asciidoc @@ -12,7 +12,8 @@ This file is generated! See scripts/docs_collector.py This is a module for ingesting data from the different Microsoft Products. Currently supports these filesets: -- `defender_atp` fileset: Supports Microsoft Defender ATP +- `defender_atp` fileset: Supports Microsoft Defender for Endpoint (Microsoft Defender ATP) +- `m365_defender` fileset: Supports Microsoft 365 Defender (Microsoft Threat Protection) - `dhcp` fileset: Supports Microsoft DHCP logs include::../include/what-happens.asciidoc[] @@ -25,6 +26,84 @@ include::../include/configuring-intro.asciidoc[] include::../include/config-option-intro.asciidoc[] +[float] +==== `m365_defender` fileset settings + +beta[] + +To configure access for Filebeat to Microsoft 365 Defender you will have to create a new Azure Application registration, this will again return Oauth tokens with access to the Microsoft 365 Defender API + +The procedure to create an application is found on the below link: + +https://docs.microsoft.com/en-us/microsoft-365/security/mtp/api-create-app-web?view=o365-worldwide#create-an-app[Create a new Azure Application] + +When giving the application the API permissions described in the documentation (Incident.Read.All) it will only grant access to read Incidents from 365 Defender and nothing else in the Azure Domain. + +After the application has been created, it should contain 3 values that you need to apply to the module configuration. + +These values are: + +- Client ID +- Client Secret +- Tenant ID + +Example config: + +[source,yaml] +---- +- module: microsoft + m365_defender: + enabled: true + var.oauth2.client.id: "123abc-879546asd-349587-ad64508" + var.oauth2.client.secret: "980453~-Sg99gedf" + var.oauth2.token_url: "https://login.microsoftonline.com/INSERT-TENANT-ID/oauth2/token" +---- + +*`var.oauth2.client.id`*:: + +This is the client ID related to creating a new application on Azure. + +*`var.oauth2.client.secret`*:: + +The secret related to the client ID. + +*`var.oauth2.token_url`*:: + +A predefined URL towards the Oauth2 service for Microsoft. The URL should always be the same with the exception of the Tenant ID that needs to be added to the full URL. + +[float] +==== 365 Defender ECS fields + +This is a list of 365 Defender fields that are mapped to ECS. + +[options="header"] +|====================================================================== +| 365 Defender Fields | ECS Fields | +| lastUpdateTime | @timestamp | +| severity | event.severity | +| createdTime | event.created | +| alerts.category | threat.technique.name | +| alerts.description | rule.description | +| alerts.serviceSource | event.provider | +| alerts.alertId | event.id | +| alerts.firstActivity | event.start | +| alerts.lastActivity | event.end | +| alerts.title | message | +| entities.processId | process.pid | +| entities.processCommandLine | process.command_line | +| entities.processCreationTime | process.start | +| entities.parentProcessId | process.parent.pid | +| entities.parentProcessCreationTime | process.parent.start | +| entities.sha1 | file.hash.sha1 | +| entities.sha256 | file.hash.sha256 | +| entities.url | url.full | +| entities.filePath | file.path | +| entities.fileName | file.name | +| entities.userPrincipalName | host.user.name | +| entities.domainName | host.user.domain | +| entities.aadUserId | host.user.id | +|====================================================================== + [float] ==== `defender_atp` fileset settings @@ -114,7 +193,7 @@ This module comes with a sample dashboard for Defender ATP. [role="screenshot"] image::./images/filebeat-defender-atp-overview.png[] -The best way to view Defender ATP events and alert data is in the SIEM. +The best way to view Defender ATP events and alert data is in the SIEM. [role="screenshot"] image::./images/siem-alerts-cs.jpg[] diff --git a/filebeat/docs/reload-configuration.asciidoc b/filebeat/docs/reload-configuration.asciidoc index 10a706d4020..14c4826b6d8 100644 --- a/filebeat/docs/reload-configuration.asciidoc +++ b/filebeat/docs/reload-configuration.asciidoc @@ -33,9 +33,11 @@ definitions. TIP: The first line of each external configuration file must be an input definition that starts with `- type`. Make sure you omit the line -+{beatname_lc}.config.inputs+ from this file. - -For example: ++{beatname_lc}.config.inputs+ from this file. All <> +must be specified within each external configuration file. Specifying these +configuration options at the global `filebeat.config.inputs` level is not supported. + +Example external configuration file: [source,yaml] ------------------------------------------------------------------------------ diff --git a/filebeat/docs/running-on-cloudfoundry.asciidoc b/filebeat/docs/running-on-cloudfoundry.asciidoc index ae9603dc012..5aa3ef95837 100644 --- a/filebeat/docs/running-on-cloudfoundry.asciidoc +++ b/filebeat/docs/running-on-cloudfoundry.asciidoc @@ -66,16 +66,8 @@ To check the status, run: $ cf apps name requested state instances memory disk urls -filebeat started 1/1 256M 1G +filebeat started 1/1 512M 1G ------------------------------------------------ Log events should start flowing to Elasticsearch. The events are annotated with metadata added by the <> processor. - - -[WARNING] -======================================= -*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that -{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` -will result in duplicate events being pushed to Elasticsearch. -======================================= diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index bf29e0715ed..3a0d1d5d19c 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -398,6 +398,7 @@ filebeat.inputs: # # Possible options are: # * log: Reads every line of the log file (default) +# * filestream: Improved version of log input. Experimental. # * stdin: Reads the standard in #------------------------------ Log input -------------------------------- @@ -618,6 +619,145 @@ filebeat.inputs: # Defines if inputs is enabled #enabled: true +#--------------------------- Filestream input ---------------------------- +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + # To fetch all ".log" files from a specific level of subdirectories + # /var/log/*/*.log can be used. + # For each file found under this path, a harvester is started. + # Make sure not file is defined twice as this can lead to unexpected behaviour. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Configure the file encoding for reading files with international characters + # following the W3C recommendation for HTML5 (http://www.w3.org/TR/encoding). + # Some sample encodings: + # plain, utf-8, utf-16be-bom, utf-16be, utf-16le, big5, gb18030, gbk, + # hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ... + #encoding: plain + + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, no lines are dropped. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, all the lines are exported. + #include_lines: ['^ERR', '^WARN'] + + ### Prospector options + + # How often the input checks for new files in the paths that are specified + # for harvesting. Specify 1s to scan the directory as frequently as possible + # without causing Filebeat to scan too frequently. Default: 10s. + #prospector.scanner.check_interval: 10s + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Expand "**" patterns into regular glob patterns. + #prospector.scanner.recursive_glob: true + + # If symlinks is enabled, symlinks are opened and harvested. The harvester is opening the + # original for harvesting but will report the symlink name as source. + #prospector.scanner.symlinks: false + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + #clean_inactive: 0 + + # Removes the state for file which cannot be found on disk anymore immediately + #clean_removed: true + + # Method to determine if two files are the same or not. By default + # the Beat considers two files the same if their inode and device id are the same. + #file_identity.native: ~ + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + # Set to true to publish fields with null values in events. + #keep_null: false + + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + + # Ignore files which were modified more then the defined timespan in the past. + # ignore_older is disabled by default, so no files are ignored by setting it to 0. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #ignore_older: 0 + + # Defines the buffer size every harvester uses when fetching the file + #harvester_buffer_size: 16384 + + # Maximum number of bytes a single log event can have + # All bytes after max_bytes are discarded and not sent. The default is 10MB. + # This is especially useful for multiline log messages which can get large. + #message_max_bytes: 10485760 + + # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + #line_terminator: auto + + # The Ingest Node pipeline ID associated with this input. If this is set, it + # overwrites the pipeline option from the Elasticsearch output. + #pipeline: + + # Backoff values define how aggressively filebeat crawls new files for updates + # The default values can be used in most cases. Backoff defines how long it is waited + # to check a file again after EOF is reached. Default is 1s which means the file + # is checked every second if new lines were added. This leads to a near real time crawling. + # Every time a new line appears, backoff is reset to the initial value. + #backoff.init: 1s + + # Max backoff defines what the maximum backoff time is. After having backed off multiple times + # from checking the files, the waiting time will never exceed max_backoff independent of the + # backoff factor. Having it set to 10s means in the worst case a new line can be added to a log + # file after having backed off multiple times, it takes a maximum of 10s to read the new line + #backoff.max: 10s + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #close.on_state_change.inactive: 5m + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.on_state_change.renamed: false + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + #close.on_state_change.removed: true + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.eof: false + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.after_interval: 0 + #----------------------------- Stdin input ------------------------------- # Configuration to use stdin input #- type: stdin diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index 2a3e678c542..9a29bc76962 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -62,6 +62,35 @@ filebeat.inputs: # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after +# filestream is an experimental input. It is going to replace log input in the future. +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + # ============================== Filebeat modules ============================== filebeat.config.modules: diff --git a/filebeat/input/default-inputs/inputs.go b/filebeat/input/default-inputs/inputs.go index 52338a0af98..881f3664efd 100644 --- a/filebeat/input/default-inputs/inputs.go +++ b/filebeat/input/default-inputs/inputs.go @@ -19,6 +19,7 @@ package inputs import ( "github.com/elastic/beats/v7/filebeat/beater" + "github.com/elastic/beats/v7/filebeat/input/filestream" "github.com/elastic/beats/v7/filebeat/input/unix" v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" @@ -27,13 +28,14 @@ import ( func Init(info beat.Info, log *logp.Logger, components beater.StateStore) []v2.Plugin { return append( - genericInputs(), + genericInputs(log, components), osInputs(info, log, components)..., ) } -func genericInputs() []v2.Plugin { +func genericInputs(log *logp.Logger, components beater.StateStore) []v2.Plugin { return []v2.Plugin{ + filestream.Plugin(log, components), unix.Plugin(), } } diff --git a/filebeat/input/filestream/config.go b/filebeat/input/filestream/config.go index 93b23232594..5b582ccf6e8 100644 --- a/filebeat/input/filestream/config.go +++ b/filebeat/input/filestream/config.go @@ -30,10 +30,11 @@ import ( // Config stores the options of a file stream. type config struct { + readerConfig + Paths []string `config:"paths"` Close closerConfig `config:"close"` - FileWatcher *common.ConfigNamespace `config:"file_watcher"` - Reader readerConfig `config:"readers"` + FileWatcher *common.ConfigNamespace `config:"prospector"` FileIdentity *common.ConfigNamespace `config:"file_identity"` CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"` CleanRemoved bool `config:"clean_removed"` @@ -47,18 +48,17 @@ type closerConfig struct { } type readerCloserConfig struct { - AfterInterval time.Duration - Inactive time.Duration - OnEOF bool + AfterInterval time.Duration `config:"after_interval"` + OnEOF bool `config:"on_eof"` } type stateChangeCloserConfig struct { - CheckInterval time.Duration - Removed bool - Renamed bool + CheckInterval time.Duration `config:"check_interval" validate:"nonzero"` + Inactive time.Duration `config:"inactive"` + Removed bool `config:"removed"` + Renamed bool `config:"renamed"` } -// TODO should this be inline? type readerConfig struct { Backoff backoffConfig `config:"backoff"` BufferSize int `config:"buffer_size"` @@ -79,9 +79,9 @@ type backoffConfig struct { func defaultConfig() config { return config{ + readerConfig: defaultReaderConfig(), Paths: []string{}, Close: defaultCloserConfig(), - Reader: defaultReaderConfig(), CleanInactive: 0, CleanRemoved: true, HarvesterLimit: 0, @@ -94,11 +94,11 @@ func defaultCloserConfig() closerConfig { OnStateChange: stateChangeCloserConfig{ CheckInterval: 5 * time.Second, Removed: true, // TODO check clean_removed option + Inactive: 0 * time.Second, Renamed: false, }, Reader: readerCloserConfig{ OnEOF: false, - Inactive: 0 * time.Second, AfterInterval: 0 * time.Second, }, } diff --git a/filebeat/input/filestream/filestream.go b/filebeat/input/filestream/filestream.go index 59f26ccca1b..4d42bbf6242 100644 --- a/filebeat/input/filestream/filestream.go +++ b/filebeat/input/filestream/filestream.go @@ -26,6 +26,7 @@ import ( input "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/common/backoff" + "github.com/elastic/beats/v7/libbeat/common/file" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/go-concert/ctxtool" "github.com/elastic/go-concert/unison" @@ -43,10 +44,14 @@ type logFile struct { ctx context.Context cancelReading context.CancelFunc - closeInactive time.Duration closeAfterInterval time.Duration closeOnEOF bool + checkInterval time.Duration + closeInactive time.Duration + closeRemoved bool + closeRenamed bool + offset int64 lastTimeRead time.Time backoff backoff.Backoff @@ -59,7 +64,7 @@ func newFileReader( canceler input.Canceler, f *os.File, config readerConfig, - closerConfig readerCloserConfig, + closerConfig closerConfig, ) (*logFile, error) { offset, err := f.Seek(0, os.SEEK_CUR) if err != nil { @@ -69,9 +74,12 @@ func newFileReader( l := &logFile{ file: f, log: log, - closeInactive: closerConfig.Inactive, - closeAfterInterval: closerConfig.AfterInterval, - closeOnEOF: closerConfig.OnEOF, + closeAfterInterval: closerConfig.Reader.AfterInterval, + closeOnEOF: closerConfig.Reader.OnEOF, + checkInterval: closerConfig.OnStateChange.CheckInterval, + closeInactive: closerConfig.OnStateChange.Inactive, + closeRemoved: closerConfig.OnStateChange.Removed, + closeRenamed: closerConfig.OnStateChange.Renamed, offset: offset, lastTimeRead: time.Now(), backoff: backoff.NewExpBackoff(canceler.Done(), config.Backoff.Init, config.Backoff.Max), @@ -143,7 +151,7 @@ func (f *logFile) startFileMonitoringIfNeeded() { if f.closeAfterInterval > 0 { f.tg.Go(func(ctx unison.Canceler) error { - f.closeIfInactive(ctx) + f.periodicStateCheck(ctx) return nil }) } @@ -164,10 +172,8 @@ func (f *logFile) closeIfTimeout(ctx unison.Canceler) { } } -func (f *logFile) closeIfInactive(ctx unison.Canceler) { - // This can be made configureble if users need a more flexible - // cheking for inactive files. - ticker := time.NewTicker(5 * time.Minute) +func (f *logFile) periodicStateCheck(ctx unison.Canceler) { + ticker := time.NewTicker(f.checkInterval) defer ticker.Stop() for { @@ -175,8 +181,7 @@ func (f *logFile) closeIfInactive(ctx unison.Canceler) { case <-ctx.Done(): return case <-ticker.C: - age := time.Since(f.lastTimeRead) - if age > f.closeInactive { + if f.shouldBeClosed() { f.cancelReading() return } @@ -184,6 +189,52 @@ func (f *logFile) closeIfInactive(ctx unison.Canceler) { } } +func (f *logFile) shouldBeClosed() bool { + if f.closeInactive > 0 { + if time.Since(f.lastTimeRead) > f.closeInactive { + return true + } + } + + if !f.closeRemoved && !f.closeRenamed { + return false + + } + + info, statErr := f.file.Stat() + if statErr != nil { + f.log.Errorf("Unexpected error reading from %s; error: %s", f.file.Name(), statErr) + return true + } + + if f.closeRenamed { + // Check if the file can still be found under the same path + if !isSameFile(f.file.Name(), info) { + f.log.Debugf("close_renamed is enabled and file %s has been renamed", f.file.Name()) + return true + } + } + + if f.closeRemoved { + // Check if the file name exists. See https://github.com/elastic/filebeat/issues/93 + if file.IsRemoved(f.file) { + f.log.Debugf("close_removed is enabled and file %s has been removed", f.file.Name()) + return true + } + } + + return false +} + +func isSameFile(path string, info os.FileInfo) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + + return os.SameFile(fileInfo, info) +} + // errorChecks determines the cause for EOF errors, and how the EOF event should be handled // based on the config options. func (f *logFile) errorChecks(err error) error { diff --git a/filebeat/input/filestream/fswatch.go b/filebeat/input/filestream/fswatch.go index d4bc1b5ea08..e988fb3cee9 100644 --- a/filebeat/input/filestream/fswatch.go +++ b/filebeat/input/filestream/fswatch.go @@ -57,9 +57,9 @@ type fileScanner struct { type fileWatcherConfig struct { // Interval is the time between two scans. - Interval time.Duration + Interval time.Duration `config:"check_interval"` // Scanner is the configuration of the scanner. - Scanner fileScannerConfig + Scanner fileScannerConfig `config:",inline"` } // fileWatcher gets the list of files from a FSWatcher and creates events by @@ -74,7 +74,7 @@ type fileWatcher struct { func newFileWatcher(paths []string, ns *common.ConfigNamespace) (loginp.FSWatcher, error) { if ns == nil { - return newScannerWatcher(paths, nil) + return newScannerWatcher(paths, common.NewConfig()) } watcherType := ns.Name() @@ -212,10 +212,9 @@ func (w *fileWatcher) Event() loginp.FSEvent { } type fileScannerConfig struct { - Paths []string - ExcludedFiles []match.Matcher - Symlinks bool - RecursiveGlob bool + ExcludedFiles []match.Matcher `config:"exclude_files"` + Symlinks bool `config:"symlinks"` + RecursiveGlob bool `config:"recursive_glob"` } func defaultFileScannerConfig() fileScannerConfig { diff --git a/filebeat/input/filestream/fswatch_test.go b/filebeat/input/filestream/fswatch_test.go index 5e63987c868..2435fad1500 100644 --- a/filebeat/input/filestream/fswatch_test.go +++ b/filebeat/input/filestream/fswatch_test.go @@ -31,6 +31,12 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) +var ( + excludedFilePath = filepath.Join("testdata", "excluded_file") + includedFilePath = filepath.Join("testdata", "included_file") + directoryPath = filepath.Join("testdata", "unharvestable_dir") +) + func TestFileScanner(t *testing.T) { testCases := map[string]struct { paths []string @@ -39,56 +45,30 @@ func TestFileScanner(t *testing.T) { expectedFiles []string }{ "select all files": { - paths: []string{ - filepath.Join("testdata", "excluded_file"), - filepath.Join("testdata", "included_file"), - }, + paths: []string{excludedFilePath, includedFilePath}, expectedFiles: []string{ - mustAbsPath(filepath.Join("testdata", "excluded_file")), - mustAbsPath(filepath.Join("testdata", "included_file")), + mustAbsPath(excludedFilePath), + mustAbsPath(includedFilePath), }, }, "skip excluded files": { - paths: []string{ - filepath.Join("testdata", "excluded_file"), - filepath.Join("testdata", "included_file"), - }, + paths: []string{excludedFilePath, includedFilePath}, excludedFiles: []match.Matcher{ - match.MustCompile(filepath.Join("testdata", "excluded_file")), + match.MustCompile("excluded_file"), }, expectedFiles: []string{ - mustAbsPath(filepath.Join("testdata", "included_file")), - }, - }, - // covers test_input.py/test_skip_symlinks - "skip symlinks": { - paths: []string{ - filepath.Join("testdata", "symlink_to_included_file"), - filepath.Join("testdata", "included_file"), - }, - symlinks: false, - expectedFiles: []string{ - mustAbsPath(filepath.Join("testdata", "included_file")), - }, - }, - "return a file once if symlinks are enabled": { - paths: []string{ - filepath.Join("testdata", "symlink_to_included_file"), - filepath.Join("testdata", "included_file"), - }, - symlinks: true, - expectedFiles: []string{ - mustAbsPath(filepath.Join("testdata", "included_file")), + mustAbsPath(includedFilePath), }, }, "skip directories": { - paths: []string{ - filepath.Join("testdata", "unharvestable_dir"), - }, + paths: []string{directoryPath}, expectedFiles: []string{}, }, } + setupFilesForScannerTest(t) + defer removeFilesOfScannerTest(t) + for name, test := range testCases { test := test @@ -107,11 +87,33 @@ func TestFileScanner(t *testing.T) { for p, _ := range files { paths = append(paths, p) } - assert.Equal(t, test.expectedFiles, paths) + assert.ElementsMatch(t, paths, test.expectedFiles) }) } } +func setupFilesForScannerTest(t *testing.T) { + err := os.MkdirAll(directoryPath, 0750) + if err != nil { + t.Fatal(t) + } + + for _, path := range []string{excludedFilePath, includedFilePath} { + f, err := os.Create(path) + if err != nil { + t.Fatalf("file %s, error %v", path, err) + } + f.Close() + } +} + +func removeFilesOfScannerTest(t *testing.T) { + err := os.RemoveAll("testdata") + if err != nil { + t.Fatal(err) + } +} + func TestFileWatchNewDeleteModified(t *testing.T) { oldTs := time.Now() newTs := oldTs.Add(5 * time.Second) @@ -201,65 +203,17 @@ func TestFileWatchNewDeleteModified(t *testing.T) { events: make(chan loginp.FSEvent), } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go w.watch(ctx) + go w.watch(context.Background()) - for _, expectedEvent := range test.expectedEvents { - evt := w.Event() - assert.Equal(t, expectedEvent, evt) + count := len(test.expectedEvents) + actual := make([]loginp.FSEvent, count) + for i := 0; i < count; i++ { + actual[i] = w.Event() } - }) - } -} -func TestFileWatcherRenamedFile(t *testing.T) { - testPath := mustAbsPath(filepath.Join("testdata", "first_name")) - renamedPath := mustAbsPath(filepath.Join("testdata", "renamed")) - - f, err := os.Create(testPath) - if err != nil { - t.Fatal(err) - } - f.Close() - fi, err := os.Stat(testPath) - if err != nil { - t.Fatal(err) - } - - cfg := fileScannerConfig{ - ExcludedFiles: nil, - Symlinks: false, - RecursiveGlob: false, - } - scanner, err := newFileScanner([]string{testPath, renamedPath}, cfg) - if err != nil { - t.Fatal(err) - } - w := fileWatcher{ - log: logp.L(), - scanner: scanner, - events: make(chan loginp.FSEvent), - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - go w.watch(ctx) - assert.Equal(t, loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: testPath, Info: fi}, w.Event()) - - err = os.Rename(testPath, renamedPath) - if err != nil { - t.Fatal(err) - } - defer os.Remove(renamedPath) - fi, err = os.Stat(renamedPath) - if err != nil { - t.Fatal(err) + assert.ElementsMatch(t, actual, test.expectedEvents) + }) } - - go w.watch(ctx) - assert.Equal(t, loginp.FSEvent{Op: loginp.OpRename, OldPath: testPath, NewPath: renamedPath, Info: fi}, w.Event()) } type mockScanner struct { diff --git a/filebeat/input/filestream/fswatch_test_non_windows.go b/filebeat/input/filestream/fswatch_test_non_windows.go new file mode 100644 index 00000000000..3c316efdfc9 --- /dev/null +++ b/filebeat/input/filestream/fswatch_test_non_windows.go @@ -0,0 +1,144 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package filestream + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common/match" + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestFileScannerSymlinks(t *testing.T) { + testCases := map[string]struct { + paths []string + excludedFiles []match.Matcher + symlinks bool + expectedFiles []string + }{ + // covers test_input.py/test_skip_symlinks + "skip symlinks": { + paths: []string{ + filepath.Join("testdata", "symlink_to_included_file"), + filepath.Join("testdata", "included_file"), + }, + symlinks: false, + expectedFiles: []string{ + mustAbsPath(filepath.Join("testdata", "included_file")), + }, + }, + "return a file once if symlinks are enabled": { + paths: []string{ + filepath.Join("testdata", "symlink_to_included_file"), + filepath.Join("testdata", "included_file"), + }, + symlinks: true, + expectedFiles: []string{ + mustAbsPath(filepath.Join("testdata", "included_file")), + }, + }, + } + + err := os.Symlink( + mustAbsPath(filepath.Join("testdata", "included_file")), + mustAbsPath(filepath.Join("testdata", "symlink_to_included_file")), + ) + if err != nil { + t.Fatal(err) + } + + for name, test := range testCases { + test := test + + t.Run(name, func(t *testing.T) { + cfg := fileScannerConfig{ + ExcludedFiles: test.excludedFiles, + Symlinks: true, + RecursiveGlob: false, + } + fs, err := newFileScanner(test.paths, cfg) + if err != nil { + t.Fatal(err) + } + files := fs.GetFiles() + paths := make([]string, 0) + for p, _ := range files { + paths = append(paths, p) + } + assert.ElementsMatch(t, test.expectedFiles, paths) + }) + } +} + +func TestFileWatcherRenamedFile(t *testing.T) { + testPath := mustAbsPath("first_name") + renamedPath := mustAbsPath("renamed") + + f, err := os.Create(testPath) + if err != nil { + t.Fatal(err) + } + f.Close() + fi, err := os.Stat(testPath) + if err != nil { + t.Fatal(err) + } + + cfg := fileScannerConfig{ + ExcludedFiles: nil, + Symlinks: false, + RecursiveGlob: false, + } + scanner, err := newFileScanner([]string{testPath, renamedPath}, cfg) + if err != nil { + t.Fatal(err) + } + w := fileWatcher{ + log: logp.L(), + scanner: scanner, + events: make(chan loginp.FSEvent), + } + + go w.watch(context.Background()) + assert.Equal(t, loginp.FSEvent{Op: loginp.OpCreate, OldPath: "", NewPath: testPath, Info: fi}, w.Event()) + + err = os.Rename(testPath, renamedPath) + if err != nil { + t.Fatal(err) + } + defer os.Remove(renamedPath) + fi, err = os.Stat(renamedPath) + if err != nil { + t.Fatal(err) + } + + go w.watch(context.Background()) + evt := w.Event() + + assert.Equal(t, loginp.OpRename, evt.Op) + assert.Equal(t, testPath, evt.OldPath) + assert.Equal(t, renamedPath, evt.NewPath) +} diff --git a/filebeat/input/filestream/identifier.go b/filebeat/input/filestream/identifier.go new file mode 100644 index 00000000000..736c66da2f0 --- /dev/null +++ b/filebeat/input/filestream/identifier.go @@ -0,0 +1,139 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package filestream + +import ( + "fmt" + "os" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/file" +) + +const ( + nativeName = "native" + pathName = "path" + inodeMarkerName = "inode_marker" + + DefaultIdentifierName = nativeName + identitySep = "::" +) + +var ( + identifierFactories = map[string]identifierFactory{ + nativeName: newINodeDeviceIdentifier, + pathName: newPathIdentifier, + inodeMarkerName: newINodeMarkerIdentifier, + } +) + +type identifierFactory func(*common.Config) (fileIdentifier, error) + +type fileIdentifier interface { + GetSource(loginp.FSEvent) fileSource + Name() string +} + +// fileSource implements the Source interface +// It is required to identify and manage file sources. +type fileSource struct { + info os.FileInfo + newPath string + oldPath string + + name string + identifierGenerator string +} + +// Name returns the registry identifier of the file. +func (f fileSource) Name() string { + return f.name +} + +// newFileIdentifier creates a new state identifier for a log input. +func newFileIdentifier(ns *common.ConfigNamespace) (fileIdentifier, error) { + if ns == nil { + return newINodeDeviceIdentifier(nil) + } + + identifierType := ns.Name() + f, ok := identifierFactories[identifierType] + if !ok { + return nil, fmt.Errorf("no such file_identity generator: %s", identifierType) + } + + return f(ns.Config()) +} + +type inodeDeviceIdentifier struct { + name string +} + +func newINodeDeviceIdentifier(_ *common.Config) (fileIdentifier, error) { + return &inodeDeviceIdentifier{ + name: nativeName, + }, nil +} + +func (i *inodeDeviceIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: pluginName + identitySep + i.name + identitySep + file.GetOSState(e.Info).String(), + identifierGenerator: i.name, + } +} + +func (i *inodeDeviceIdentifier) Name() string { + return i.name +} + +type pathIdentifier struct { + name string +} + +func newPathIdentifier(_ *common.Config) (fileIdentifier, error) { + return &pathIdentifier{ + name: pathName, + }, nil +} + +func (p *pathIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: pluginName + identitySep + p.name + identitySep + e.NewPath, + identifierGenerator: p.name, + } +} + +func (p *pathIdentifier) Name() string { + return p.name +} + +// mockIdentifier is used for testing +type MockIdentifier struct{} + +func (m *MockIdentifier) GetSource(e loginp.FSEvent) fileSource { + return fileSource{identifierGenerator: "mock"} +} + +func (m *MockIdentifier) Name() string { return "mock" } diff --git a/filebeat/input/filestream/identifier_inode_deviceid.go b/filebeat/input/filestream/identifier_inode_deviceid.go new file mode 100644 index 00000000000..25254d97fdc --- /dev/null +++ b/filebeat/input/filestream/identifier_inode_deviceid.go @@ -0,0 +1,108 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package filestream + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/file" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type inodeMarkerIdentifier struct { + log *logp.Logger + name string + markerPath string + + markerFileLastModifitaion time.Time + markerTxt string +} + +func newINodeMarkerIdentifier(cfg *common.Config) (fileIdentifier, error) { + var config struct { + MarkerPath string `config:"path" validate:"required"` + } + err := cfg.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err) + } + + fi, err := os.Stat(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err) + } + markerContent, err := ioutil.ReadFile(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err) + } + return &inodeMarkerIdentifier{ + log: logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)), + name: inodeMarkerName, + markerPath: config.MarkerPath, + markerFileLastModifitaion: fi.ModTime(), + markerTxt: string(markerContent), + }, nil +} + +func (i *inodeMarkerIdentifier) markerContents() string { + f, err := os.Open(i.markerPath) + if err != nil { + i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err) + return "" + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err) + return "" + } + if i.markerFileLastModifitaion.Before(fi.ModTime()) { + contents, err := ioutil.ReadFile(i.markerPath) + if err != nil { + i.log.Errorf("Error while reading contents of marker file: %v", err) + return "" + } + i.markerTxt = string(contents) + } + + return i.markerTxt +} + +func (i *inodeMarkerIdentifier) GetSource(e loginp.FSEvent) fileSource { + osstate := file.GetOSState(e.Info) + return fileSource{ + info: e.Info, + newPath: e.NewPath, + oldPath: e.OldPath, + name: fmt.Sprintf("%s%s%s-%s", i.name, identitySep, osstate.InodeString(), i.markerContents()), + identifierGenerator: i.name, + } +} + +func (i *inodeMarkerIdentifier) Name() string { + return i.name +} diff --git a/filebeat/input/filestream/identifier_inode_deviceid_windows.go b/filebeat/input/filestream/identifier_inode_deviceid_windows.go new file mode 100644 index 00000000000..4ee8d866124 --- /dev/null +++ b/filebeat/input/filestream/identifier_inode_deviceid_windows.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package filestream + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func newINodeMarkerIdentifier(cfg *common.Config) (fileIdentifier, error) { + return nil, fmt.Errorf("inode_deviceid is not supported on Windows") +} diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 487a5f01c2a..9f715d1183e 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -18,18 +18,48 @@ package filestream import ( + "fmt" + "os" + + "golang.org/x/text/transform" + + "github.com/elastic/go-concert/ctxtool" + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/match" "github.com/elastic/beats/v7/libbeat/feature" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/reader" + "github.com/elastic/beats/v7/libbeat/reader/debug" + "github.com/elastic/beats/v7/libbeat/reader/readfile" + "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" ) +const pluginName = "filestream" + +type state struct { + Source string `json:"source" struct:"source"` + Offset int64 `json:"offset" struct:"offset"` + IdentifierName string `json:"identifier_name" struct:"identifier_name"` +} + // filestream is the input for reading from files which // are actively written by other applications. -type filestream struct{} - -const pluginName = "filestream" +type filestream struct { + readerConfig readerConfig + bufferSize int + tailFile bool // TODO + encodingFactory encoding.EncodingFactory + encoding encoding.Encoding + lineTerminator readfile.LineTerminator + excludeLines []match.Matcher + includeLines []match.Matcher + maxBytes int + closerConfig closerConfig +} // Plugin creates a new filestream input plugin for creating a stateful input. func Plugin(log *logp.Logger, store loginp.StateStore) input.Plugin { @@ -49,13 +79,46 @@ func Plugin(log *logp.Logger, store loginp.StateStore) input.Plugin { } func configure(cfg *common.Config) (loginp.Prospector, loginp.Harvester, error) { - panic("TODO: implement me") + config := defaultConfig() + if err := cfg.Unpack(&config); err != nil { + return nil, nil, err + } + + prospector, err := newFileProspector( + config.Paths, + config.IgnoreOlder, + config.FileWatcher, + config.FileIdentity, + ) + if err != nil { + return nil, nil, err + } + + encodingFactory, ok := encoding.FindEncoding(config.Encoding) + if !ok || encodingFactory == nil { + return nil, nil, fmt.Errorf("unknown encoding('%v')", config.Encoding) + } + + return prospector, &filestream{ + readerConfig: config.readerConfig, + bufferSize: config.BufferSize, + encodingFactory: encodingFactory, + lineTerminator: config.LineTerminator, + excludeLines: config.ExcludeLines, + includeLines: config.IncludeLines, + maxBytes: config.MaxBytes, + closerConfig: config.Close, + }, nil } func (inp *filestream) Name() string { return pluginName } func (inp *filestream) Test(src loginp.Source, ctx input.TestContext) error { - panic("TODO: implement me") + reader, err := inp.open(ctx.Logger, ctx.Cancelation, state{}) + if err != nil { + return err + } + return reader.Close() } func (inp *filestream) Run( @@ -64,5 +127,241 @@ func (inp *filestream) Run( cursor loginp.Cursor, publisher loginp.Publisher, ) error { - panic("TODO: implement me") + fs, ok := src.(fileSource) + if !ok { + return fmt.Errorf("not file source") + } + + log := ctx.Logger.With("path", fs.newPath).With("state-id", src.Name()) + state := initState(log, cursor, fs) + + r, err := inp.open(log, ctx.Cancelation, state) + if err != nil { + log.Errorf("File could not be opened for reading: %v", err) + return err + } + + _, streamCancel := ctxtool.WithFunc(ctxtool.FromCanceller(ctx.Cancelation), func() { + log.Debug("Closing reader of filestream") + err := r.Close() + if err != nil { + log.Errorf("Error stopping filestream reader %v", err) + } + }) + defer streamCancel() + + return inp.readFromSource(ctx, log, r, fs.newPath, state, publisher) +} + +func initState(log *logp.Logger, c loginp.Cursor, s fileSource) state { + state := state{Source: s.newPath, IdentifierName: s.identifierGenerator} + if c.IsNew() { + return state + } + + err := c.Unpack(&state) + if err != nil { + log.Error("Cannot serialize cursor data into file state: %+v", err) + } + + return state +} + +func (inp *filestream) open(log *logp.Logger, canceler input.Canceler, s state) (reader.Reader, error) { + f, err := inp.openFile(s.Source, s.Offset) + if err != nil { + return nil, err + } + + log.Debug("newLogFileReader with config.MaxBytes:", inp.maxBytes) + + // TODO: NewLineReader uses additional buffering to deal with encoding and testing + // for new lines in input stream. Simple 8-bit based encodings, or plain + // don't require 'complicated' logic. + logReader, err := newFileReader(log, canceler, f, inp.readerConfig, inp.closerConfig) + if err != nil { + return nil, err + } + + dbgReader, err := debug.AppendReaders(logReader) + if err != nil { + f.Close() + return nil, err + } + + // Configure MaxBytes limit for EncodeReader as multiplied by 4 + // for the worst case scenario where incoming UTF32 charchers are decoded to the single byte UTF-8 characters. + // This limit serves primarily to avoid memory bload or potential OOM with expectedly long lines in the file. + // The further size limiting is performed by LimitReader at the end of the readers pipeline as needed. + encReaderMaxBytes := inp.maxBytes * 4 + + var r reader.Reader + r, err = readfile.NewEncodeReader(dbgReader, readfile.Config{ + Codec: inp.encoding, + BufferSize: inp.bufferSize, + Terminator: inp.lineTerminator, + MaxBytes: encReaderMaxBytes, + }) + if err != nil { + f.Close() + return nil, err + } + + r = readfile.NewStripNewline(r, inp.lineTerminator) + r = readfile.NewLimitReader(r, inp.maxBytes) + + return r, nil +} + +// openFile opens a file and checks for the encoding. In case the encoding cannot be detected +// or the file cannot be opened because for example of failing read permissions, an error +// is returned and the harvester is closed. The file will be picked up again the next time +// the file system is scanned +func (inp *filestream) openFile(path string, offset int64) (*os.File, error) { + err := inp.checkFileBeforeOpening(path) + if err != nil { + return nil, err + } + + f, err := os.OpenFile(path, os.O_RDONLY, os.FileMode(0)) + if err != nil { + return nil, fmt.Errorf("failed opening %s: %s", path, err) + } + + err = inp.initFileOffset(f, offset) + if err != nil { + f.Close() + return nil, err + } + + inp.encoding, err = inp.encodingFactory(f) + if err != nil { + f.Close() + if err == transform.ErrShortSrc { + return nil, fmt.Errorf("initialising encoding for '%v' failed due to file being too short", f) + } + return nil, fmt.Errorf("initialising encoding for '%v' failed: %v", f, err) + } + + return f, nil +} + +func (inp *filestream) checkFileBeforeOpening(path string) error { + fi, err := os.Stat(path) + if err != nil { + return fmt.Errorf("failed to stat source file %s: %v", path, err) + } + + if !fi.Mode().IsRegular() { + return fmt.Errorf("tried to open non regular file: %q %s", fi.Mode(), fi.Name()) + } + + if fi.Mode()&os.ModeNamedPipe != 0 { + return fmt.Errorf("failed to open file %s, named pipes are not supported", path) + } + + return nil +} + +func (inp *filestream) initFileOffset(file *os.File, offset int64) error { + if offset > 0 { + _, err := file.Seek(offset, os.SEEK_SET) + return err + } + + // get offset from file in case of encoding factory was required to read some data. + _, err := file.Seek(0, os.SEEK_CUR) + return err +} + +func (inp *filestream) readFromSource( + ctx input.Context, + log *logp.Logger, + r reader.Reader, + path string, + s state, + p loginp.Publisher, +) error { + for ctx.Cancelation.Err() == nil { + message, err := r.Next() + if err != nil { + switch err { + case ErrFileTruncate: + log.Info("File was truncated. Begin reading file from offset 0.") + s.Offset = 0 + case ErrClosed: + log.Info("Reader was closed. Closing.") + case reader.ErrLineUnparsable: + log.Info("Skipping unparsable line in file.") + continue + default: + log.Errorf("Read line error: %v", err) + } + return nil + } + + if message.IsEmpty() || inp.isDroppedLine(log, string(message.Content)) { + continue + } + + event := inp.eventFromMessage(message, path) + s.Offset += int64(message.Bytes) + + if err := p.Publish(event, s); err != nil { + return err + } + } + return nil +} + +// isDroppedLine decides if the line is exported or not based on +// the include_lines and exclude_lines options. +func (inp *filestream) isDroppedLine(log *logp.Logger, line string) bool { + if len(inp.includeLines) > 0 { + if !matchAny(inp.includeLines, line) { + log.Debug("Drop line as it does not match any of the include patterns %s", line) + return true + } + } + if len(inp.excludeLines) > 0 { + if matchAny(inp.excludeLines, line) { + log.Debug("Drop line as it does match one of the exclude patterns%s", line) + return true + } + } + + return false +} + +func matchAny(matchers []match.Matcher, text string) bool { + for _, m := range matchers { + if m.MatchString(text) { + return true + } + } + return false +} + +func (inp *filestream) eventFromMessage(m reader.Message, path string) beat.Event { + fields := common.MapStr{ + "log": common.MapStr{ + "offset": m.Bytes, // Offset here is the offset before the starting char. + "file": common.MapStr{ + "path": path, + }, + }, + } + fields.DeepUpdate(m.Fields) + + if len(m.Content) > 0 { + if fields == nil { + fields = common.MapStr{} + } + fields["message"] = string(m.Content) + } + + return beat.Event{ + Timestamp: m.Ts, + Fields: fields, + } } diff --git a/filebeat/input/filestream/prospector.go b/filebeat/input/filestream/prospector.go index 257574b9ca1..94670e18ce7 100644 --- a/filebeat/input/filestream/prospector.go +++ b/filebeat/input/filestream/prospector.go @@ -18,19 +18,191 @@ package filestream import ( + "os" + "strings" + "time" + + "github.com/urso/sderr" + loginp "github.com/elastic/beats/v7/filebeat/input/filestream/internal/input-logfile" input "github.com/elastic/beats/v7/filebeat/input/v2" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/statestore" + "github.com/elastic/go-concert/unison" +) + +const ( + prospectorDebugKey = "file_prospector" ) // fileProspector implements the Prospector interface. // It contains a file scanner which returns file system events. // The FS events then trigger either new Harvester runs or updates // the statestore. -type fileProspector struct{} +type fileProspector struct { + filewatcher loginp.FSWatcher + identifier fileIdentifier + ignoreOlder time.Duration + cleanRemoved bool +} + +func newFileProspector( + paths []string, + ignoreOlder time.Duration, + fileWatcherNs, identifierNs *common.ConfigNamespace, +) (loginp.Prospector, error) { + + filewatcher, err := newFileWatcher(paths, fileWatcherNs) + if err != nil { + return nil, err + } + + identifier, err := newFileIdentifier(identifierNs) + if err != nil { + return nil, err + } + + return &fileProspector{ + filewatcher: filewatcher, + identifier: identifier, + ignoreOlder: ignoreOlder, + cleanRemoved: true, + }, nil +} +// Run starts the fileProspector which accepts FS events from a file watcher. func (p *fileProspector) Run(ctx input.Context, s *statestore.Store, hg *loginp.HarvesterGroup) { - panic("TODO: implement me") + log := ctx.Logger.With("prospector", prospectorDebugKey) + log.Debug("Starting prospector") + defer log.Debug("Prospector has stopped") + + if p.cleanRemoved { + p.cleanRemovedBetweenRuns(log, s) + } + + p.updateIdentifiersBetweenRuns(log, s) + + var tg unison.MultiErrGroup + + tg.Go(func() error { + p.filewatcher.Run(ctx.Cancelation) + return nil + }) + + tg.Go(func() error { + for ctx.Cancelation.Err() == nil { + fe := p.filewatcher.Event() + + if fe.Op == loginp.OpDone { + return nil + } + + src := p.identifier.GetSource(fe) + switch fe.Op { + case loginp.OpCreate: + log.Debugf("A new file %s has been found", fe.NewPath) + + if p.ignoreOlder > 0 { + now := time.Now() + if now.Sub(fe.Info.ModTime()) > p.ignoreOlder { + log.Debugf("Ignore file because ignore_older reached. File %s", fe.NewPath) + break + } + } + + hg.Run(ctx, src) + + case loginp.OpWrite: + log.Debugf("File %s has been updated", fe.NewPath) + + hg.Run(ctx, src) + + case loginp.OpDelete: + log.Debugf("File %s has been removed", fe.OldPath) + + if p.cleanRemoved { + log.Debugf("Remove state for file as file removed: %s", fe.OldPath) + + err := s.Remove(src.Name()) + if err != nil { + log.Errorf("Error while removing state from statestore: %v", err) + } + } + + case loginp.OpRename: + log.Debugf("File %s has been renamed to %s", fe.OldPath, fe.NewPath) + // TODO update state information in the store + + default: + log.Error("Unkown return value %v", fe.Op) + } + } + return nil + }) + + errs := tg.Wait() + if len(errs) > 0 { + log.Error("%s", sderr.WrapAll(errs, "running prospector failed")) + } +} + +func (p *fileProspector) cleanRemovedBetweenRuns(log *logp.Logger, s *statestore.Store) { + keyPrefix := pluginName + "::" + s.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", + key, err) + return true, nil + } + + _, err := os.Stat(st.Source) + if err != nil { + s.Remove(key) + } + + return true, nil + }) +} + +func (p *fileProspector) updateIdentifiersBetweenRuns(log *logp.Logger, s *statestore.Store) { + keyPrefix := pluginName + "::" + s.Each(func(key string, dec statestore.ValueDecoder) (bool, error) { + if !strings.HasPrefix(string(key), keyPrefix) { + return true, nil + } + + var st state + if err := dec.Decode(&st); err != nil { + log.Errorf("Failed to read regisry state for '%v', cursor state will be ignored. Error was: %+v", key, err) + return true, nil + } + + if st.IdentifierName == p.identifier.Name() { + return true, nil + } + + fi, err := os.Stat(st.Source) + if err != nil { + return true, nil + } + newKey := p.identifier.GetSource(loginp.FSEvent{NewPath: st.Source, Info: fi}).Name() + st.IdentifierName = p.identifier.Name() + + err = s.Set(newKey, st) + if err != nil { + log.Errorf("Failed to add updated state for '%v', cursor state will be ignored. Error was: %+v", key, err) + return true, nil + } + s.Remove(key) + + return true, nil + }) } func (p *fileProspector) Test() error { diff --git a/filebeat/input/filestream/testdata/excluded_file b/filebeat/input/filestream/testdata/excluded_file deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/filebeat/input/filestream/testdata/included_file b/filebeat/input/filestream/testdata/included_file deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/filebeat/input/filestream/testdata/symlink_to_included_file b/filebeat/input/filestream/testdata/symlink_to_included_file deleted file mode 120000 index 40824f3f7d3..00000000000 --- a/filebeat/input/filestream/testdata/symlink_to_included_file +++ /dev/null @@ -1 +0,0 @@ -filebeat/input/filestream/testdata/included_file \ No newline at end of file diff --git a/filebeat/inputsource/unix/server_test.go b/filebeat/inputsource/unix/server_test.go index a9043d14a8e..fc9545100d5 100644 --- a/filebeat/inputsource/unix/server_test.go +++ b/filebeat/inputsource/unix/server_test.go @@ -60,6 +60,10 @@ func TestErrorOnEmptyLineDelimiter(t *testing.T) { } func TestReceiveEventsAndMetadata(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test is only supported on non-windows. See https://github.com/elastic/beats/issues/19641") + return + } expectedMessages := generateMessages(5, 100) largeMessages := generateMessages(10, 4096) extraLargeMessages := generateMessages(2, 65*1024) @@ -255,6 +259,10 @@ func TestSocketOwnershipAndMode(t *testing.T) { } func TestSocketCleanup(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test is only supported on non-windows. See https://github.com/elastic/beats/issues/21757") + return + } path := filepath.Join(os.TempDir(), "test.sock") mockStaleSocket, err := net.Listen("unix", path) require.NoError(t, err) @@ -298,6 +306,10 @@ func TestSocketCleanupRefusal(t *testing.T) { } func TestReceiveNewEventsConcurrently(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test is only supported on non-windows. See https://github.com/elastic/beats/issues/21757") + return + } workers := 4 eventsCount := 100 path := filepath.Join(os.TempDir(), "test.sock") diff --git a/filebeat/module/apache/access/config/access.yml b/filebeat/module/apache/access/config/access.yml index 183de629867..b39221031f3 100644 --- a/filebeat/module/apache/access/config/access.yml +++ b/filebeat/module/apache/access/config/access.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/apache/error/config/error.yml b/filebeat/module/apache/error/config/error.yml index 7aa0cdb7cf8..b8aa75eef7c 100644 --- a/filebeat/module/apache/error/config/error.yml +++ b/filebeat/module/apache/error/config/error.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/auditd/log/config/log.yml b/filebeat/module/auditd/log/config/log.yml index 183de629867..b39221031f3 100644 --- a/filebeat/module/auditd/log/config/log.yml +++ b/filebeat/module/auditd/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/audit/config/audit.yml b/filebeat/module/elasticsearch/audit/config/audit.yml index 7aa0cdb7cf8..b8aa75eef7c 100644 --- a/filebeat/module/elasticsearch/audit/config/audit.yml +++ b/filebeat/module/elasticsearch/audit/config/audit.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/deprecation/config/log.yml b/filebeat/module/elasticsearch/deprecation/config/log.yml index f6c37b9e426..aa6f7d220f0 100644 --- a/filebeat/module/elasticsearch/deprecation/config/log.yml +++ b/filebeat/module/elasticsearch/deprecation/config/log.yml @@ -15,4 +15,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/gc/config/gc.yml b/filebeat/module/elasticsearch/gc/config/gc.yml index 5f62e00c54c..12b9a0f874d 100644 --- a/filebeat/module/elasticsearch/gc/config/gc.yml +++ b/filebeat/module/elasticsearch/gc/config/gc.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/server/config/log.yml b/filebeat/module/elasticsearch/server/config/log.yml index b0b9e3f55d0..45a7f84018d 100644 --- a/filebeat/module/elasticsearch/server/config/log.yml +++ b/filebeat/module/elasticsearch/server/config/log.yml @@ -15,4 +15,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml index 2d98286de6d..4c861ea7dc7 100644 --- a/filebeat/module/elasticsearch/slowlog/config/slowlog.yml +++ b/filebeat/module/elasticsearch/slowlog/config/slowlog.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/haproxy/log/config/file.yml b/filebeat/module/haproxy/log/config/file.yml index c3e104f56d4..ac3846087c3 100644 --- a/filebeat/module/haproxy/log/config/file.yml +++ b/filebeat/module/haproxy/log/config/file.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/haproxy/log/config/syslog.yml b/filebeat/module/haproxy/log/config/syslog.yml index 6fa56f8fe3c..e919d732e74 100644 --- a/filebeat/module/haproxy/log/config/syslog.yml +++ b/filebeat/module/haproxy/log/config/syslog.yml @@ -6,4 +6,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/debug/config/debug.yml b/filebeat/module/icinga/debug/config/debug.yml index 33c47850878..8b5c2ae7422 100644 --- a/filebeat/module/icinga/debug/config/debug.yml +++ b/filebeat/module/icinga/debug/config/debug.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/main/config/main.yml b/filebeat/module/icinga/main/config/main.yml index 33c47850878..8b5c2ae7422 100644 --- a/filebeat/module/icinga/main/config/main.yml +++ b/filebeat/module/icinga/main/config/main.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/icinga/startup/config/startup.yml b/filebeat/module/icinga/startup/config/startup.yml index a69343bd796..acc8bd2b782 100644 --- a/filebeat/module/icinga/startup/config/startup.yml +++ b/filebeat/module/icinga/startup/config/startup.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/iis/access/config/iis-access.yml b/filebeat/module/iis/access/config/iis-access.yml index b4797039397..2c845646a4e 100644 --- a/filebeat/module/iis/access/config/iis-access.yml +++ b/filebeat/module/iis/access/config/iis-access.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/iis/error/config/iis-error.yml b/filebeat/module/iis/error/config/iis-error.yml index b4797039397..2c845646a4e 100644 --- a/filebeat/module/iis/error/config/iis-error.yml +++ b/filebeat/module/iis/error/config/iis-error.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/kafka/log/config/log.yml b/filebeat/module/kafka/log/config/log.yml index a745c79562c..f3cfc687526 100644 --- a/filebeat/module/kafka/log/config/log.yml +++ b/filebeat/module/kafka/log/config/log.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/kibana/log/config/log.yml b/filebeat/module/kibana/log/config/log.yml index 1ad204c62fe..93daa116791 100644 --- a/filebeat/module/kibana/log/config/log.yml +++ b/filebeat/module/kibana/log/config/log.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/logstash/log/config/log.yml b/filebeat/module/logstash/log/config/log.yml index af0e4c33735..407078dc115 100644 --- a/filebeat/module/logstash/log/config/log.yml +++ b/filebeat/module/logstash/log/config/log.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/logstash/slowlog/config/slowlog.yml b/filebeat/module/logstash/slowlog/config/slowlog.yml index 080a2c4310d..4c6abc278c6 100644 --- a/filebeat/module/logstash/slowlog/config/slowlog.yml +++ b/filebeat/module/logstash/slowlog/config/slowlog.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mongodb/log/config/log.yml b/filebeat/module/mongodb/log/config/log.yml index 183de629867..b39221031f3 100644 --- a/filebeat/module/mongodb/log/config/log.yml +++ b/filebeat/module/mongodb/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mysql/error/config/error.yml b/filebeat/module/mysql/error/config/error.yml index 818cbab297d..8ceba377810 100644 --- a/filebeat/module/mysql/error/config/error.yml +++ b/filebeat/module/mysql/error/config/error.yml @@ -16,4 +16,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/mysql/slowlog/config/slowlog.yml b/filebeat/module/mysql/slowlog/config/slowlog.yml index 7b1a3bc28fd..6893491d869 100644 --- a/filebeat/module/mysql/slowlog/config/slowlog.yml +++ b/filebeat/module/mysql/slowlog/config/slowlog.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nats/log/config/log.yml b/filebeat/module/nats/log/config/log.yml index 183de629867..b39221031f3 100644 --- a/filebeat/module/nats/log/config/log.yml +++ b/filebeat/module/nats/log/config/log.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/access/config/nginx-access.yml b/filebeat/module/nginx/access/config/nginx-access.yml index 7aa0cdb7cf8..b8aa75eef7c 100644 --- a/filebeat/module/nginx/access/config/nginx-access.yml +++ b/filebeat/module/nginx/access/config/nginx-access.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/error/config/nginx-error.yml b/filebeat/module/nginx/error/config/nginx-error.yml index 797c45f6c38..173a43c5a1e 100644 --- a/filebeat/module/nginx/error/config/nginx-error.yml +++ b/filebeat/module/nginx/error/config/nginx-error.yml @@ -14,4 +14,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml index 7aa0cdb7cf8..b8aa75eef7c 100644 --- a/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml +++ b/filebeat/module/nginx/ingress_controller/config/ingress_controller.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/postgresql/log/config/log.yml b/filebeat/module/postgresql/log/config/log.yml index f69d3e4d387..eeac43780bf 100644 --- a/filebeat/module/postgresql/log/config/log.yml +++ b/filebeat/module/postgresql/log/config/log.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/redis/log/config/log.yml b/filebeat/module/redis/log/config/log.yml index 9d6cae46b38..1c6b6d6c084 100644 --- a/filebeat/module/redis/log/config/log.yml +++ b/filebeat/module/redis/log/config/log.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/module/traefik/access/config/traefik-access.yml b/filebeat/module/traefik/access/config/traefik-access.yml index 183de629867..b39221031f3 100644 --- a/filebeat/module/traefik/access/config/traefik-access.yml +++ b/filebeat/module/traefik/access/config/traefik-access.yml @@ -8,4 +8,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index d449258c40f..fa3caa93475 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -131,22 +131,29 @@ def run_on_file(self, module, fileset, test_file, cfgfile): cmd.append("{module}.{fileset}.var.format=json".format(module=module, fileset=fileset)) output_path = os.path.join(self.working_dir) - output = open(os.path.join(output_path, "output.log"), "ab") - output.write(bytes(" ".join(cmd) + "\n", "utf-8")) - - # Use a fixed timezone so results don't vary depending on the environment - # Don't use UTC to avoid hiding that non-UTC timezones are not being converted as needed, - # this can happen because UTC uses to be the default timezone in date parsers when no other - # timezone is specified. - local_env = os.environ.copy() - local_env["TZ"] = 'Etc/GMT+2' - - subprocess.Popen(cmd, - env=local_env, - stdin=None, - stdout=output, - stderr=subprocess.STDOUT, - bufsize=0).wait() + # Runs inside a with block to ensure file is closed afterwards + with open(os.path.join(output_path, "output.log"), "ab") as output: + output.write(bytes(" ".join(cmd) + "\n", "utf-8")) + + # Use a fixed timezone so results don't vary depending on the environment + # Don't use UTC to avoid hiding that non-UTC timezones are not being converted as needed, + # this can happen because UTC uses to be the default timezone in date parsers when no other + # timezone is specified. + local_env = os.environ.copy() + local_env["TZ"] = 'Etc/GMT+2' + + subprocess.Popen(cmd, + env=local_env, + stdin=None, + stdout=output, + stderr=subprocess.STDOUT, + bufsize=0).wait() + + # List of errors to check in filebeat output logs + errors = ["Error loading pipeline for fileset"] + # Checks if the output of filebeat includes errors + contains_error, error_line = file_contains(os.path.join(output_path, "output.log"), errors) + assert contains_error is False, "Error found in log:{}".format(error_line) # Make sure index exists self.wait_until(lambda: self.es.indices.exists(self.index_name)) @@ -305,5 +312,14 @@ def delete_key(obj, key): del obj[key] +def file_contains(filepath, strings): + with open(filepath, 'r') as file: + for line in file: + for string in strings: + if string in line: + return True, line + return False, None + + def pretty_json(obj): return json.dumps(obj, indent=2, separators=(',', ': ')) diff --git a/go.mod b/go.mod index 09cd086cbee..720690f1f2f 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/davecgh/go-xdr v0.0.0-20161123171359-e6a2ba005892 // indirect github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 // indirect + github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible // indirect github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 github.com/dlclark/regexp2 v1.1.7-0.20171009020623-7632a260cbaf // indirect @@ -55,7 +56,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/dop251/goja v0.0.0-20200831102558-9af81ddcf0e1 github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 - github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 + github.com/dustin/go-humanize v1.0.0 github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200121105743-0d940dd29fd2 github.com/elastic/ecs v1.6.0 github.com/elastic/elastic-agent-client/v7 v7.0.0-20200709172729-d43b7ad5833a @@ -142,12 +143,13 @@ require ( github.com/satori/go.uuid v1.2.0 // indirect github.com/shirou/gopsutil v2.19.11+incompatible github.com/shopspring/decimal v1.2.0 - github.com/spf13/cobra v0.0.3 + github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.6.1 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 + github.com/ugorji/go/codec v1.1.8 github.com/urso/sderr v0.0.0-20200210124243-c2a16f3d43ec github.com/vmware/govmomi v0.0.0-20170802214208-2cad15190b41 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c diff --git a/go.sum b/go.sum index 031f1faa095..5c01c612fe3 100644 --- a/go.sum +++ b/go.sum @@ -76,6 +76,8 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5 h1:ygIc8M6trr62pF5DucadTWGdEB4mEyvzi0e2nbcmcyA= @@ -83,6 +85,8 @@ github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tT github.com/Microsoft/hcsshim v0.8.7 h1:ptnOoufxGSzauVTsdE+wMYnCWA301PdoN4xg5oRdZpg= github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= @@ -112,6 +116,7 @@ github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d h1:OE3kzLBpy7pOJEzE55 github.com/antlr/antlr4 v0.0.0-20200820155224-be881fa6b91d/go.mod h1:T7PbCXFs94rrTttyxjbyT5+/1V8T2TYDejxUfHJjw1Y= github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77 h1:afT88tB6u9JCKQZVAAaa9ICz/uGn5Uw9ekn6P22mYKM= github.com/apoydence/eachers v0.0.0-20181020210610-23942921fe77/go.mod h1:bXvGk6IkT1Agy7qzJ+DjIw/SJ1AaB3AvAuMDVV+Vkoo= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= @@ -141,6 +146,8 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e h1:Gbx+iVCXG/1m5WSnidDGuHgN+vbIwl+6fR092ANU+Y8= github.com/cavaliercoder/go-rpm v0.0.0-20190131055624-7a9c54e3d83e/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -170,12 +177,16 @@ github.com/containerd/fifo v0.0.0-20190816180239-bda0ff6ed73c/go.mod h1:ODA38xgv github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea h1:n2Ltr3SrfQlf/9nOna1DoGKxLx3qTSI8Ttl6Xrqp6mw= github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cucumber/godog v0.8.1 h1:lVb+X41I4YDreE+ibZ50bdXmySxgRviYFgKY6Aw4XE8= github.com/cucumber/godog v0.8.1/go.mod h1:vSh3r/lM+psC1BPXvdkSEuNjmXfpVqrMGYAElF6hxnA= github.com/cyphar/filepath-securejoin v0.2.2 h1:jCwT2GTP+PY5nBz3c/YL5PAIbusElVrPujOBSCj8xRg= @@ -190,9 +201,15 @@ github.com/denisenkom/go-mssqldb v0.0.0-20200206145737-bbfc9a55622e/go.mod h1:xb github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2 h1:6+hM8KeYKV0Z9EIINNqIEDyyIRAcNc2FW+/TUYNmWyw= github.com/devigned/tab v0.1.2-0.20190607222403-0c15cf42f9a2/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b h1:mUDs72Rlzv6A4YN8w3Ra3hU9x/plOQPcQjZYL/1f5SM= +github.com/dgraph-io/badger/v2 v2.2007.3-0.20201012072640-f5a7e0a1c83b/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible h1:4jGdduO4ceTJFKf0IhgaB8NJapGqKHwC2b4xQ/cXujM= github.com/dgrijalva/jwt-go v3.2.1-0.20190620180102-5e25c22bd5d6+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1 h1:eG5K5GNAAHvQlFmfIuy0Ocjg5dvyX22g/KknwTpmBko= github.com/digitalocean/go-libvirt v0.0.0-20180301200012-6075ea3c39a1/go.mod h1:PRcPVAAma6zcLpFd4GZrjR/MRpood3TamjKI2m/z/Uw= github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= @@ -213,8 +230,9 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96 h1:cenwrSVm+Z7QL github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6 h1:RrkoB0pT3gnjXhL/t10BSP1mcr/0Ldea2uMyuBr2SWk= github.com/dop251/goja_nodejs v0.0.0-20171011081505-adff31b136e6/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= -github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= @@ -327,7 +345,6 @@ github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -339,7 +356,6 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -366,7 +382,6 @@ github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -403,7 +418,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.13.0 h1:sBDQoHXrOlfPobnKw69FIKa1wg9qsL github.com/grpc-ecosystem/grpc-gateway v1.13.0/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao= github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce h1:prjrVgOk2Yg6w+PflHoszQNLTUh4kaByUcEWM/9uin4= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -424,6 +438,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d h1:Ft6PtvobE9vwkCsuoNO5DZDbhKkKuktAlSsiOi1X5NA= github.com/hashicorp/golang-lru v0.5.2-0.20190520140433-59383c442f7d/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01 h1:HiJF8Mek+I7PY0Bm+SuhkwaAZSZP83sw6rrTMrgZ0io= github.com/haya14busa/go-actions-toolkit v0.0.0-20200105081403-ca0307860f01/go.mod h1:1DWDZmeYf0LX30zscWb7K9rUMeirNeBMd5Dum+seUhc= github.com/haya14busa/go-checkstyle v0.0.0-20170303121022-5e9d09f51fa1/go.mod h1:RsN5RGgVYeXpcXNtWyztD5VIe7VNSEqpJvF2iEH7QvI= @@ -455,7 +470,6 @@ github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgb github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -478,7 +492,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -490,10 +503,10 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01 h1:EPw7R3OAyxHBCyl0oqh3lUZqS5lu3KSxzzGasE0opXQ= github.com/lib/pq v1.1.2-0.20190507191818-2ff3cb3adc01/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/magefile/mage v1.9.0 h1:t3AU2wNwehMCW97vuqQLtw6puppWXHO+O2MHo5a50XE= github.com/magefile/mage v1.9.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1 h1:mdxE1MF9o53iCb2Ghj1VfWvh7ZOwHpnVG/xwXrV90U8= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= @@ -543,7 +556,6 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.5.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -575,6 +587,7 @@ github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.4.1+incompatible h1:mFe7ttWaflA46Mhqh+jUfjp2qTbPYxLB2/OyBppH9dg= github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -621,6 +634,7 @@ github.com/reviewdog/reviewdog v0.9.17 h1:MKb3rlQZgkEXr3d85iqtYNITXn7gDJr2kT0Ihg github.com/reviewdog/reviewdog v0.9.17/go.mod h1:Y0yPFDTi9L5ohkoecJdgbvAhq+dUXp+zI7atqVibwKg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e h1:hUGyBE/4CXRPThr4b6kt+f1CN90no4Fs5CNrYOKYSIg= github.com/samuel/go-parser v0.0.0-20130731160455-ca8abbf65d0e/go.mod h1:Sb6li54lXV0yYEjI4wX8cucdQ9gqUJV3+Ngg3l9g30I= github.com/samuel/go-thrift v0.0.0-20140522043831-2187045faa54 h1:jbchLJWyhKcmOjkbC4zDvT/n5EEd7g6hnnF760rEyRA= @@ -650,14 +664,22 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykE github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= @@ -666,9 +688,7 @@ github.com/stretchr/testify v1.1.5-0.20170601210322-f6abca593680/go.mod h1:a8OnR github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw= github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -677,6 +697,11 @@ github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxp github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw= github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786 h1:B/IVHYiI0d04dudYw+CvCAGqSMq8d0yWy56eD6p85BQ= github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786/go.mod h1:RIkfovP3Y7my19aXEjjbNd9E5TlHozzAyt7B8AaEcwg= +github.com/ugorji/go v1.1.8 h1:/D9x7IRpfMHDlizVOgxrag5Fh+/NY+LtI8bsr+AswRA= +github.com/ugorji/go v1.1.8/go.mod h1:0lNM99SwWUIRhCXnigEMClngXBk/EmpTXa7mgiewYWA= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v1.1.8 h1:4dryPvxMP9OtkjIbuNeK2nb27M38XMHLGlfNSNph/5s= +github.com/ugorji/go/codec v1.1.8/go.mod h1:X00B19HDtwvKbQY2DcYjvZxKQp8mzrJoQ6EgoIY/D2E= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797 h1:OHNw/6pXODJAB32NujjdQO/KIYQ3KAbHQfCzH81XdCs= github.com/urso/diag v0.0.0-20200210123136-21b3cc8eb797/go.mod h1:pNWFTeQ+V1OYT/TzWpnWb6eQBdoXpdx+H+lrH97/Oyo= @@ -705,9 +730,9 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1: github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56 h1:yhqBHs09SmmUoNOHc9jgK4a60T3XFRtPAkYxVnqgY50= github.com/xeipuuv/gojsonschema v0.0.0-20181112162635-ac52e6811b56/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7 h1:0gYLpmzecnaDCoeWxSfEJ7J1b6B/67+NV++4HKQXx+Y= github.com/yuin/gopher-lua v0.0.0-20170403160031-b402f3114ec7/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= -go.elastic.co/apm v1.7.2 h1:0nwzVIPp4PDBXSYYtN19+1W5V+sj+C25UjqxDVoKcA8= go.elastic.co/apm v1.7.2/go.mod h1:tCw6CkOJgkWnzEthFN9HUP1uL3Gjc/Ur6m7gRPLaoH0= go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b h1:Sf+V3eV91ZuXjF3824SABFgXU+z4ZEuIX5ikDvt2lCE= go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b/go.mod h1:qoOSi09pnzJDh5fKnfY7bPmQgl8yl2tULdOu03xhui0= @@ -717,7 +742,6 @@ go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn go.elastic.co/apm/module/apmhttp v1.7.2/go.mod h1:sTFWiWejnhSdZv6+dMgxGec2Nxe/ZKfHfz/xtRM+cRY= go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 h1:NjYJ/beChqugXSavTkH5tF6shvr/is8jdgJ331wfwT8= go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= -go.elastic.co/fastjson v1.0.0 h1:ooXV/ABvf+tBul26jcVViPT3sBir0PvXgibYB1IQQzg= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= @@ -741,6 +765,7 @@ go.uber.org/zap v1.14.0 h1:/pduUoebOeeJzTDFuoMgC6nRkiasr1sBCIEorly7m4o= go.uber.org/zap v1.14.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -767,7 +792,6 @@ golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367 h1:0IiAsCRByjO2QjX7ZPkw5oU9x+n1YqRL802rjC0c3Aw= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= @@ -801,7 +825,6 @@ golang.org/x/net v0.0.0-20191021144547-ec77196f6094/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -818,7 +841,6 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -827,6 +849,7 @@ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -840,6 +863,7 @@ golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190529164535-6a60838ec259/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -848,12 +872,9 @@ golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c h1:OYFUffxXPezb7BVTx9AaD4Vl0qtxmklBIkwCKH1YwDY= golang.org/x/sys v0.0.0-20200102141924-c96a22e43c9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e h1:LwyF2AFISC9nVbS6MgzsaQNSUsRXI49GS+YQ5KX/QH0= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -904,7 +925,6 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= @@ -916,7 +936,6 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= -gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -948,7 +967,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 3830e053b1f..c054be0cd53 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -45,3 +45,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test heartbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/heartbeat/docs/fields.asciidoc b/heartbeat/docs/fields.asciidoc index 20e797faf1a..e80238bca4c 100644 --- a/heartbeat/docs/fields.asciidoc +++ b/heartbeat/docs/fields.asciidoc @@ -361,8 +361,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + diff --git a/heartbeat/docs/getting-started.asciidoc b/heartbeat/docs/getting-started.asciidoc index 21da1fb0547..1cd73dbb04f 100644 --- a/heartbeat/docs/getting-started.asciidoc +++ b/heartbeat/docs/getting-started.asciidoc @@ -96,9 +96,43 @@ was started. Heartbeat adds the `@every` keyword to the syntax provided by the include::{libbeat-dir}/shared/config-check.asciidoc[] +[float] +[[configurelocation]] +=== Step 4: Configure the Heartbeat location + +Heartbeat can be deployed in multiple locations so that you can detect +differences in availability and response times across those locations. +Configure the Heartbeat location to allow {kib} to display location-specific +information on Uptime maps and perform Uptime anomaly detection based +on location. + +To configure the location of a Heartbeat instance, modify the +`add_observer_metadata` processor in +{beatname_lc}.yml+. The following +example specifies the `geo.name` of the `add_observer_metadata` processor as +`us-east-1a`: + +[source,yaml] +---------------------------------------------------------------------- +# ============================ Processors ============================ + +processors: + - add_observer_metadata: + # Optional, but recommended geo settings for the location Heartbeat is running in + geo: <1> + # Token describing this location + name: us-east-1a <2> + # Lat, Lon " + #location: "37.926868, -78.024902" <3> +---------------------------------------------------------------------- +<1> Uncomment the `geo` setting. +<2> Uncomment `name` and assign the name of the location of the Heartbeat server. +<3> Optionally uncomment `location` and assign the latitude and longitude. + +include::{libbeat-dir}/shared/config-check.asciidoc[] + [float] [[setup-assets]] -=== Step 4: Set up assets +=== Step 5: Set up assets {beatname_uc} comes with predefined assets for parsing, indexing, and visualizing your data. To load these assets: @@ -128,7 +162,7 @@ environment. If you're using a different output, such as {ls}, see [float] [[start]] -=== Step 5: Start Heartbeat +=== Step 6: Start Heartbeat Before starting {beatname_uc}, modify the user credentials in +{beatname_lc}.yml+ and specify a user who is @@ -145,7 +179,7 @@ events to your defined output. [float] [[view-data]] -=== Step 6: View your data in {kib} +=== Step 7: View your data in {kib} {beatname_uc} comes with pre-built {kib} dashboards and UIs for visualizing the status of your services. The dashboards are available in the diff --git a/journalbeat/docs/fields.asciidoc b/journalbeat/docs/fields.asciidoc index bb7627508a4..57f97a4162e 100644 --- a/journalbeat/docs/fields.asciidoc +++ b/journalbeat/docs/fields.asciidoc @@ -914,8 +914,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index 699c96d8b62..f13b3c39ff2 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -138,6 +138,7 @@ type ClientEventer interface { type ProcessorList interface { Processor + Close() error All() []Processor } diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go index ca58b9b321f..714b266ea84 100644 --- a/libbeat/cmd/instance/beat.go +++ b/libbeat/cmd/instance/beat.go @@ -371,6 +371,11 @@ func (b *Beat) launch(settings Settings, bt beat.Creator) error { if err != nil { return err } + defer func() { + if err := b.processing.Close(); err != nil { + logp.Warn("Failed to close global processing: %v", err) + } + }() // Windows: Mark service as stopped. // After this is run, a Beat service is considered by the OS to be stopped diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go index 0543d37e9c3..2421c232eee 100644 --- a/libbeat/common/docker/watcher.go +++ b/libbeat/common/docker/watcher.go @@ -138,6 +138,7 @@ func NewWatcher(log *logp.Logger, host string, tls *TLSConfig, storeShortID bool // Extra check to confirm that Docker is available _, err = client.Info(context.Background()) if err != nil { + client.Close() return nil, err } @@ -395,14 +396,12 @@ func (w *watcher) cleanupWorker() { log := w.log for { - // Wait a full period - time.Sleep(w.cleanupTimeout) - select { case <-w.ctx.Done(): w.stopped.Done() return - default: + // Wait a full period + case <-time.After(w.cleanupTimeout): // Check entries for timeout var toDelete []string timeout := time.Now().Add(-w.cleanupTimeout) diff --git a/libbeat/common/transport/tlscommon/tls.go b/libbeat/common/transport/tlscommon/tls.go index 3616a9f07e4..ba44310727c 100644 --- a/libbeat/common/transport/tlscommon/tls.go +++ b/libbeat/common/transport/tlscommon/tls.go @@ -66,7 +66,7 @@ func LoadCertificate(config *CertificateConfig) (*tls.Certificate, error) { return nil, err } - log.Debugf("tls", "loading certificate: %v and key %v", certificate, key) + log.Debugf("Loading certificate: %v and key %v", certificate, key) return &cert, nil } @@ -169,7 +169,7 @@ func LoadCertificateAuthorities(CAs []string) (*x509.CertPool, []error) { errors = append(errors, fmt.Errorf("%v adding %v to the list of known CAs", ErrNotACertificate, r)) continue } - log.Debugf("tls", "successfully loaded CA certificate: %v", r) + log.Debugf("Successfully loaded CA certificate: %v", r) } return roots, errors diff --git a/libbeat/docs/shared-securing-beat.asciidoc b/libbeat/docs/shared-securing-beat.asciidoc index b8dcc3b1957..e1c47d91f2c 100644 --- a/libbeat/docs/shared-securing-beat.asciidoc +++ b/libbeat/docs/shared-securing-beat.asciidoc @@ -29,11 +29,13 @@ For secure communication between APM Server and APM Agents, see <> endif::[] +endif::[] // APM HTTPS information ifdef::beat-specific-security[] @@ -70,5 +72,7 @@ endif::[] // Linux Seccomp ifndef::serverless[] +ifndef::win_only[] include::./security/linux-seccomp.asciidoc[] endif::[] +endif::[] diff --git a/libbeat/docs/shared/obs-apps.asciidoc b/libbeat/docs/shared/obs-apps.asciidoc index 9b5f7354ea0..26698700c21 100644 --- a/libbeat/docs/shared/obs-apps.asciidoc +++ b/libbeat/docs/shared/obs-apps.asciidoc @@ -38,13 +38,13 @@ endif::[] |=== |Elastic apps | Use to -|{kibana-ref}/xpack-infra.html[{metrics-app}] +|{observability-guide}/analyze-metrics.html[{metrics-app}] |Explore metrics about systems and services across your ecosystem -|{kibana-ref}/xpack-logs.html[{logs-app}] +|{observability-guide}/monitor-logs.html[{logs-app}] |Tail related log data in real time -|{kibana-ref}/xpack-uptime.html[{uptime-app}] +|{observability-guide}/monitor-uptime.html[{uptime-app}] |Monitor availability issues across your apps and services |{kibana-ref}/xpack-apm.html[APM app] diff --git a/libbeat/logp/logger.go b/libbeat/logp/logger.go index 6f1c42fe022..2bbe6b3ce53 100644 --- a/libbeat/logp/logger.go +++ b/libbeat/logp/logger.go @@ -50,6 +50,12 @@ func NewLogger(selector string, options ...LogOption) *Logger { return newLogger(loadLogger().rootLogger, selector, options...) } +// WithOptions returns a clone of l with options applied. +func (l *Logger) WithOptions(options ...LogOption) *Logger { + cloned := l.logger.WithOptions(options...) + return &Logger{cloned, cloned.Sugar()} +} + // With creates a child logger and adds structured context to it. Fields added // to the child don't affect the parent, and vice versa. func (l *Logger) With(args ...interface{}) *Logger { diff --git a/libbeat/logp/logger_test.go b/libbeat/logp/logger_test.go new file mode 100644 index 00000000000..eaf8a1070ce --- /dev/null +++ b/libbeat/logp/logger_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package logp + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" +) + +func TestLoggerWithOptions(t *testing.T) { + core1, observed1 := observer.New(zapcore.DebugLevel) + core2, observed2 := observer.New(zapcore.DebugLevel) + + logger1 := NewLogger("bo", zap.WrapCore(func(in zapcore.Core) zapcore.Core { + return zapcore.NewTee(in, core1) + })) + logger2 := logger1.WithOptions(zap.WrapCore(func(in zapcore.Core) zapcore.Core { + return zapcore.NewTee(in, core2) + })) + + logger1.Info("hello logger1") // should just go to the first observer + logger2.Info("hello logger1 and logger2") // should go to both observers + + observedEntries1 := observed1.All() + require.Len(t, observedEntries1, 2) + assert.Equal(t, "hello logger1", observedEntries1[0].Message) + assert.Equal(t, "hello logger1 and logger2", observedEntries1[1].Message) + + observedEntries2 := observed2.All() + require.Len(t, observedEntries2, 1) + assert.Equal(t, "hello logger1 and logger2", observedEntries2[0].Message) +} diff --git a/metricbeat/module/system/diskio/disk_performance_386.go b/libbeat/metric/system/diskio/disk_performance_386.go similarity index 100% rename from metricbeat/module/system/diskio/disk_performance_386.go rename to libbeat/metric/system/diskio/disk_performance_386.go diff --git a/metricbeat/module/system/diskio/disk_performance_amd64.go b/libbeat/metric/system/diskio/disk_performance_amd64.go similarity index 100% rename from metricbeat/module/system/diskio/disk_performance_amd64.go rename to libbeat/metric/system/diskio/disk_performance_amd64.go diff --git a/metricbeat/module/system/diskio/diskstat.go b/libbeat/metric/system/diskio/diskstat.go similarity index 88% rename from metricbeat/module/system/diskio/diskstat.go rename to libbeat/metric/system/diskio/diskstat.go index 949d69778fa..3cea37c552a 100644 --- a/metricbeat/module/system/diskio/diskstat.go +++ b/libbeat/metric/system/diskio/diskstat.go @@ -25,11 +25,11 @@ import ( sigar "github.com/elastic/gosigar" ) -// mapping fields which output by `iostat -x` on linux +// IOMetric contains mapping fields which are outputed by `iostat -x` on linux // // Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await r_await w_await svctm %util // sda 0.06 0.78 0.09 0.27 9.42 8.06 48.64 0.00 1.34 0.99 1.45 0.77 0.03 -type DiskIOMetric struct { +type IOMetric struct { ReadRequestMergeCountPerSec float64 `json:"rrqmCps"` WriteRequestMergeCountPerSec float64 `json:"wrqmCps"` ReadRequestCountPerSec float64 `json:"rrqCps"` @@ -46,8 +46,9 @@ type DiskIOMetric struct { BusyPct float64 `json:"busy"` } -type DiskIOStat struct { +// IOStat carries disk statistics for all devices +type IOStat struct { lastDiskIOCounters map[string]disk.IOCountersStat - lastCpu sigar.Cpu - curCpu sigar.Cpu + lastCPU sigar.Cpu + curCPU sigar.Cpu } diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/libbeat/metric/system/diskio/diskstat_linux.go similarity index 55% rename from metricbeat/module/system/diskio/diskstat_linux.go rename to libbeat/metric/system/diskio/diskstat_linux.go index 4ecfdf700ac..826aed78c27 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/libbeat/metric/system/diskio/diskstat_linux.go @@ -26,7 +26,8 @@ import ( "github.com/elastic/beats/v7/libbeat/metric/system/cpu" ) -func Get_CLK_TCK() uint32 { +// GetCLKTCK emulates the _SC_CLK_TCK syscall +func GetCLKTCK() uint32 { // return uint32(C.sysconf(C._SC_CLK_TCK)) // NOTE: _SC_CLK_TCK should be fetched from sysconf using cgo return uint32(100) @@ -38,77 +39,78 @@ func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { } // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling creates current cpu sampling // need call as soon as get IOCounters. -func (stat *DiskIOStat) OpenSampling() error { - return stat.curCpu.Get() +func (stat *IOStat) OpenSampling() error { + return stat.curCPU.Get() } -// CalIOStatistics calculates IO statistics. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { +// CalcIOStatistics calculates IO statistics. +func (stat *IOStat) CalcIOStatistics(counter disk.IOCountersStat) (IOMetric, error) { var last disk.IOCountersStat var ok bool // if last counter not found, create one and return all 0 if last, ok = stat.lastDiskIOCounters[counter.Name]; !ok { stat.lastDiskIOCounters[counter.Name] = counter - return nil + return IOMetric{}, nil } // calculate the delta ms between the CloseSampling and OpenSampling - deltams := 1000.0 * float64(stat.curCpu.Total()-stat.lastCpu.Total()) / float64(cpu.NumCores) / float64(Get_CLK_TCK()) + deltams := 1000.0 * float64(stat.curCPU.Total()-stat.lastCPU.Total()) / float64(cpu.NumCores) / float64(GetCLKTCK()) if deltams <= 0 { - return errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") + return IOMetric{}, errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") } - rd_ios := counter.ReadCount - last.ReadCount - rd_merges := counter.MergedReadCount - last.MergedReadCount - rd_bytes := counter.ReadBytes - last.ReadBytes - rd_ticks := counter.ReadTime - last.ReadTime - wr_ios := counter.WriteCount - last.WriteCount - wr_merges := counter.MergedWriteCount - last.MergedWriteCount - wr_bytes := counter.WriteBytes - last.WriteBytes - wr_ticks := counter.WriteTime - last.WriteTime + rdIOs := counter.ReadCount - last.ReadCount + rdMerges := counter.MergedReadCount - last.MergedReadCount + rdBytes := counter.ReadBytes - last.ReadBytes + rdTicks := counter.ReadTime - last.ReadTime + wrIOs := counter.WriteCount - last.WriteCount + wrMerges := counter.MergedWriteCount - last.MergedWriteCount + wrBytes := counter.WriteBytes - last.WriteBytes + wrTicks := counter.WriteTime - last.WriteTime ticks := counter.IoTime - last.IoTime aveq := counter.WeightedIO - last.WeightedIO - n_ios := rd_ios + wr_ios - n_ticks := rd_ticks + wr_ticks - n_bytes := rd_bytes + wr_bytes + nIOs := rdIOs + wrIOs + nTicks := rdTicks + wrTicks + nBytes := rdBytes + wrBytes size := float64(0) wait := float64(0) svct := float64(0) - if n_ios > 0 { - size = float64(n_bytes) / float64(n_ios) - wait = float64(n_ticks) / float64(n_ios) - svct = float64(ticks) / float64(n_ios) + if nIOs > 0 { + size = float64(nBytes) / float64(nIOs) + wait = float64(nTicks) / float64(nIOs) + svct = float64(ticks) / float64(nIOs) } queue := float64(aveq) / deltams - per_sec := func(x uint64) float64 { + perSec := func(x uint64) float64 { return 1000.0 * float64(x) / deltams } - result.ReadRequestMergeCountPerSec = per_sec(rd_merges) - result.WriteRequestMergeCountPerSec = per_sec(wr_merges) - result.ReadRequestCountPerSec = per_sec(rd_ios) - result.WriteRequestCountPerSec = per_sec(wr_ios) - result.ReadBytesPerSec = per_sec(rd_bytes) - result.WriteBytesPerSec = per_sec(wr_bytes) + result := IOMetric{} + result.ReadRequestMergeCountPerSec = perSec(rdMerges) + result.WriteRequestMergeCountPerSec = perSec(wrMerges) + result.ReadRequestCountPerSec = perSec(rdIOs) + result.WriteRequestCountPerSec = perSec(wrIOs) + result.ReadBytesPerSec = perSec(rdBytes) + result.WriteBytesPerSec = perSec(wrBytes) result.AvgRequestSize = size result.AvgQueueSize = queue result.AvgAwaitTime = wait - if rd_ios > 0 { - result.AvgReadAwaitTime = float64(rd_ticks) / float64(rd_ios) + if rdIOs > 0 { + result.AvgReadAwaitTime = float64(rdTicks) / float64(rdIOs) } - if wr_ios > 0 { - result.AvgWriteAwaitTime = float64(wr_ticks) / float64(wr_ios) + if wrIOs > 0 { + result.AvgWriteAwaitTime = float64(wrTicks) / float64(wrIOs) } result.AvgServiceTime = svct result.BusyPct = 100.0 * float64(ticks) / deltams @@ -117,10 +119,11 @@ func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCou } stat.lastDiskIOCounters[counter.Name] = counter - return nil + return result, nil } -func (stat *DiskIOStat) CloseSampling() { - stat.lastCpu = stat.curCpu +// CloseSampling closes the disk sampler +func (stat *IOStat) CloseSampling() { + stat.lastCPU = stat.curCPU } diff --git a/metricbeat/module/system/diskio/diskstat_linux_test.go b/libbeat/metric/system/diskio/diskstat_linux_test.go similarity index 55% rename from metricbeat/module/system/diskio/diskstat_linux_test.go rename to libbeat/metric/system/diskio/diskstat_linux_test.go index 56a8d9dc7cc..74c0bef231e 100644 --- a/metricbeat/module/system/diskio/diskstat_linux_test.go +++ b/libbeat/metric/system/diskio/diskstat_linux_test.go @@ -27,53 +27,11 @@ import ( "github.com/stretchr/testify/assert" sigar "github.com/elastic/gosigar" - - mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/metricbeat/module/system" ) -func Test_Get_CLK_TCK(t *testing.T) { +func Test_GetCLKTCK(t *testing.T) { //usually the tick is 100 - assert.Equal(t, uint32(100), Get_CLK_TCK()) -} - -func TestDataNameFilter(t *testing.T) { - oldFS := system.HostFS - newFS := "_meta/testdata" - system.HostFS = &newFS - defer func() { - system.HostFS = oldFS - }() - - conf := map[string]interface{}{ - "module": "system", - "metricsets": []string{"diskio"}, - "diskio.include_devices": []string{"sda", "sda1", "sda2"}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, conf) - data, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, errs) - assert.Equal(t, 3, len(data)) -} - -func TestDataEmptyFilter(t *testing.T) { - oldFS := system.HostFS - newFS := "_meta/testdata" - system.HostFS = &newFS - defer func() { - system.HostFS = oldFS - }() - - conf := map[string]interface{}{ - "module": "system", - "metricsets": []string{"diskio"}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, conf) - data, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, errs) - assert.Equal(t, 10, len(data)) + assert.Equal(t, uint32(100), GetCLKTCK()) } func TestDiskIOStat_CalIOStatistics(t *testing.T) { @@ -85,9 +43,9 @@ func TestDiskIOStat_CalIOStatistics(t *testing.T) { Name: "iostat", } - stat := &DiskIOStat{ + stat := &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{ - "iostat": disk.IOCountersStat{ + "iostat": { ReadCount: 3, WriteCount: 5, ReadTime: 7, @@ -95,17 +53,17 @@ func TestDiskIOStat_CalIOStatistics(t *testing.T) { Name: "iostat", }, }, - lastCpu: sigar.Cpu{Idle: 100}, - curCpu: sigar.Cpu{Idle: 1}, + lastCPU: sigar.Cpu{Idle: 100}, + curCPU: sigar.Cpu{Idle: 1}, } - expected := DiskIOMetric{ + expected := IOMetric{ AvgAwaitTime: 24.0 / 22.0, AvgReadAwaitTime: 1.2, AvgWriteAwaitTime: 1, } - var got DiskIOMetric - err := stat.CalIOStatistics(&got, counter) + + got, err := stat.CalcIOStatistics(counter) if err != nil { t.Fatal(err) } diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/libbeat/metric/system/diskio/diskstat_other.go similarity index 79% rename from metricbeat/module/system/diskio/diskstat_other.go rename to libbeat/metric/system/diskio/diskstat_other.go index d2669117d9e..ad5208b1b4f 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/libbeat/metric/system/diskio/diskstat_other.go @@ -25,24 +25,24 @@ import ( ) // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling stub for linux implementation. -func (stat *DiskIOStat) OpenSampling() error { +func (stat *IOStat) OpenSampling() error { return nil } -// CalIOStatistics stub for linux implementation. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("not implemented out of linux") +// CalcIOStatistics stub for linux implementation. +func (stat *IOStat) CalcIOStatistics(rcounter disk.IOCountersStat) (IOMetric, error) { + return IOMetric{}, errors.New("not implemented out of linux") } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() {} +func (stat *IOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/libbeat/metric/system/diskio/diskstat_windows.go similarity index 78% rename from metricbeat/module/system/diskio/diskstat_windows.go rename to libbeat/metric/system/diskio/diskstat_windows.go index 1e2b363074b..665caab9130 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/libbeat/metric/system/diskio/diskstat_windows.go @@ -25,24 +25,24 @@ import ( ) // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling stub for linux implementation. -func (stat *DiskIOStat) OpenSampling() error { +func (stat *IOStat) OpenSampling() error { return nil } -// CalIOStatistics stub for linux implementation. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("iostat is not implement for Windows") +// CalcIOStatistics stub for linux implementation. +func (stat *IOStat) CalcIOStatistics(counter disk.IOCountersStat) (IOMetric, error) { + return IOMetric{}, errors.New("iostat is not implement for Windows") } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() {} +func (stat *IOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/libbeat/metric/system/diskio/diskstat_windows_helper.go similarity index 100% rename from metricbeat/module/system/diskio/diskstat_windows_helper.go rename to libbeat/metric/system/diskio/diskstat_windows_helper.go diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/libbeat/metric/system/diskio/diskstat_windows_test.go similarity index 100% rename from metricbeat/module/system/diskio/diskstat_windows_test.go rename to libbeat/metric/system/diskio/diskstat_windows_test.go diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index de1a8063667..5376d563fca 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -26,6 +26,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" jsprocessor "github.com/elastic/beats/v7/libbeat/processors/script/javascript/module/processor" @@ -53,6 +54,7 @@ type addCloudMetadata struct { type initData struct { fetchers []metadataFetcher timeout time.Duration + tlsConfig *tlscommon.TLSConfig overwrite bool } @@ -63,14 +65,24 @@ func New(c *common.Config) (processors.Processor, error) { return nil, errors.Wrap(err, "failed to unpack add_cloud_metadata config") } + tlsConfig, err := tlscommon.LoadTLSConfig(config.TLS) + if err != nil { + return nil, errors.Wrap(err, "TLS configuration load") + } + initProviders := selectProviders(config.Providers, cloudMetaProviders) fetchers, err := setupFetchers(initProviders, c) if err != nil { return nil, err } p := &addCloudMetadata{ - initData: &initData{fetchers, config.Timeout, config.Overwrite}, - logger: logp.NewLogger("add_cloud_metadata"), + initData: &initData{ + fetchers: fetchers, + timeout: config.Timeout, + tlsConfig: tlsConfig, + overwrite: config.Overwrite, + }, + logger: logp.NewLogger("add_cloud_metadata"), } go p.init() diff --git a/libbeat/processors/add_cloud_metadata/config.go b/libbeat/processors/add_cloud_metadata/config.go index 08f4a241483..93a31137592 100644 --- a/libbeat/processors/add_cloud_metadata/config.go +++ b/libbeat/processors/add_cloud_metadata/config.go @@ -20,12 +20,15 @@ package add_cloud_metadata import ( "fmt" "time" + + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) type config struct { - Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. - Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. - Providers providerList `config:"providers"` // List of providers to probe + Timeout time.Duration `config:"timeout"` // Amount of time to wait for responses from the metadata services. + TLS *tlscommon.Config `config:"ssl"` // TLS configuration + Overwrite bool `config:"overwrite"` // Overwrite if cloud.* fields already exist. + Providers providerList `config:"providers"` // List of providers to probe } type providerList []string diff --git a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc index 41c0dd6d9f3..44497f31539 100644 --- a/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc +++ b/libbeat/processors/add_cloud_metadata/docs/add_cloud_metadata.asciidoc @@ -52,12 +52,16 @@ List of names the `providers` setting supports: - "aws", or "ec2" for Amazon Web Services (enabled by default). - "gcp" for Google Copmute Enging (enabled by default). - "openstack", or "nova" for Openstack Nova (enabled by default). +- "openstack-ssl", or "nova-ssl" for Openstack Nova when SSL metadata APIs are enabled (enabled by default). - "tencent", or "qcloud" for Tencent Cloud (disabled by default). The third optional configuration setting is `overwrite`. When `overwrite` is `true`, `add_cloud_metadata` overwrites existing `cloud.*` fields (`false` by default). +The `add_cloud_metadata` processor supports SSL options to configure the http +client used to query cloud metadata. See <> for more information. + The metadata that is added to events varies by hosting provider. Below are examples for each of the supported providers. diff --git a/libbeat/processors/add_cloud_metadata/http_fetcher.go b/libbeat/processors/add_cloud_metadata/http_fetcher.go index e9b18478661..e337edd5be9 100644 --- a/libbeat/processors/add_cloud_metadata/http_fetcher.go +++ b/libbeat/processors/add_cloud_metadata/http_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) type httpMetadataFetcher struct { @@ -127,9 +128,15 @@ func (f *httpMetadataFetcher) fetchRaw( // getMetadataURLs loads config and generates the metadata URLs. func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string) ([]string, error) { + return getMetadataURLsWithScheme(c, "http", defaultHost, metadataURIs) +} + +// getMetadataURLsWithScheme loads config and generates the metadata URLs. +func getMetadataURLsWithScheme(c *common.Config, scheme string, defaultHost string, metadataURIs []string) ([]string, error) { var urls []string config := struct { - MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). + MetadataHostAndPort string `config:"host"` // Specifies the host and port of the metadata service (for testing purposes only). + TLSConfig *tlscommon.Config `config:"ssl"` }{ MetadataHostAndPort: defaultHost, } @@ -138,7 +145,7 @@ func getMetadataURLs(c *common.Config, defaultHost string, metadataURIs []string return urls, errors.Wrap(err, "failed to unpack add_cloud_metadata config") } for _, uri := range metadataURIs { - urls = append(urls, "http://"+config.MetadataHostAndPort+uri) + urls = append(urls, scheme+"://"+config.MetadataHostAndPort+uri) } return urls, nil } diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index 9cd3eba55b8..3028d531c1e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -34,6 +34,9 @@ var azureVMMetadataFetcher = provider{ azHeaders := map[string]string{"Metadata": "true"} azSchema := func(m map[string]interface{}) common.MapStr { out, _ := s.Schema{ + "account": s.Object{ + "id": c.Str("subscriptionId"), + }, "instance": s.Object{ "id": c.Str("vmId"), "name": c.Str("name"), diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go index 307ac60abad..a988cc8873f 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm_test.go @@ -40,7 +40,8 @@ const azInstanceIdentityDocument = `{ "sku": "14.04.4-LTS", "version": "14.04.201605091", "vmId": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "vmSize": "Standard_D3_v2" + "vmSize": "Standard_D3_v2", + "subscriptionId": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e" }` func initAzureTestServer() *httptest.Server { @@ -87,6 +88,9 @@ func TestRetrieveAzureMetadata(t *testing.T) { "machine": common.MapStr{ "type": "Standard_D3_v2", }, + "account": common.MapStr{ + "id": "5tfb04c3-63de-4709-a9f9-9ab8c0411d5e", + }, "region": "eastus2", }, } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go index 17bc5abf689..7c9a997e0e2 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go @@ -32,16 +32,24 @@ const ( // OpenStack Nova Metadata Service // Document https://docs.openstack.org/nova/latest/user/metadata-service.html var openstackNovaMetadataFetcher = provider{ - Name: "openstack-nova", + Name: "openstack-nova", + Local: true, + Create: buildOpenstackNovaCreate("http"), +} - Local: true, +var openstackNovaSSLMetadataFetcher = provider{ + Name: "openstack-nova-ssl", + Local: true, + Create: buildOpenstackNovaCreate("https"), +} - Create: func(provider string, c *common.Config) (metadataFetcher, error) { +func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Config) (metadataFetcher, error) { + return func(provider string, c *common.Config) (metadataFetcher, error) { osSchema := func(m map[string]interface{}) common.MapStr { return common.MapStr(m) } - urls, err := getMetadataURLs(c, metadataHost, []string{ + urls, err := getMetadataURLsWithScheme(c, scheme, metadataHost, []string{ osMetadataInstanceIDURI, osMetadataInstanceTypeURI, osMetadataHostnameURI, @@ -71,5 +79,5 @@ var openstackNovaMetadataFetcher = provider{ } fetcher := &httpMetadataFetcher{"openstack", nil, responseHandlers, osSchema} return fetcher, nil - }, + } } diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go index d5c38a8437b..0a63c026cde 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova_test.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) -func initOpenstackNovaTestServer() *httptest.Server { - return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { +func openstackNovaMetadataHandler() http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.RequestURI == osMetadataInstanceIDURI { w.Write([]byte("i-0000ffac")) return @@ -49,13 +49,13 @@ func initOpenstackNovaTestServer() *httptest.Server { } http.Error(w, "not found", http.StatusNotFound) - })) + }) } func TestRetrieveOpenstackNovaMetadata(t *testing.T) { logp.TestingSetup() - server := initOpenstackNovaTestServer() + server := httptest.NewServer(openstackNovaMetadataHandler()) defer server.Close() config, err := common.NewConfigFrom(map[string]interface{}{ @@ -66,6 +66,28 @@ func TestRetrieveOpenstackNovaMetadata(t *testing.T) { t.Fatal(err) } + assertOpenstackNova(t, config) +} + +func TestRetrieveOpenstackNovaMetadataWithHTTPS(t *testing.T) { + logp.TestingSetup() + + server := httptest.NewTLSServer(openstackNovaMetadataHandler()) + defer server.Close() + + config, err := common.NewConfigFrom(map[string]interface{}{ + "host": server.Listener.Addr().String(), + "ssl.verification_mode": "none", + }) + + if err != nil { + t.Fatal(err) + } + + assertOpenstackNova(t, config) +} + +func assertOpenstackNova(t *testing.T, config *common.Config) { p, err := New(config) if err != nil { t.Fatal(err) diff --git a/libbeat/processors/add_cloud_metadata/providers.go b/libbeat/processors/add_cloud_metadata/providers.go index f93e8cf78f4..ea03b64258a 100644 --- a/libbeat/processors/add_cloud_metadata/providers.go +++ b/libbeat/processors/add_cloud_metadata/providers.go @@ -51,17 +51,19 @@ type result struct { } var cloudMetaProviders = map[string]provider{ - "alibaba": alibabaCloudMetadataFetcher, - "ecs": alibabaCloudMetadataFetcher, - "azure": azureVMMetadataFetcher, - "digitalocean": doMetadataFetcher, - "aws": ec2MetadataFetcher, - "ec2": ec2MetadataFetcher, - "gcp": gceMetadataFetcher, - "openstack": openstackNovaMetadataFetcher, - "nova": openstackNovaMetadataFetcher, - "qcloud": qcloudMetadataFetcher, - "tencent": qcloudMetadataFetcher, + "alibaba": alibabaCloudMetadataFetcher, + "ecs": alibabaCloudMetadataFetcher, + "azure": azureVMMetadataFetcher, + "digitalocean": doMetadataFetcher, + "aws": ec2MetadataFetcher, + "ec2": ec2MetadataFetcher, + "gcp": gceMetadataFetcher, + "openstack": openstackNovaMetadataFetcher, + "nova": openstackNovaMetadataFetcher, + "openstack-ssl": openstackNovaSSLMetadataFetcher, + "nova-ssl": openstackNovaSSLMetadataFetcher, + "qcloud": qcloudMetadataFetcher, + "tencent": qcloudMetadataFetcher, } func selectProviders(configList providerList, providers map[string]provider) map[string]provider { @@ -138,6 +140,7 @@ func (p *addCloudMetadata) fetchMetadata() *result { Timeout: p.initData.timeout, KeepAlive: 0, }).DialContext, + TLSClientConfig: p.initData.tlsConfig.ToConfig(), }, } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index cf0fda79d8d..beaca3bb46b 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -209,6 +209,18 @@ func (d *addDockerMetadata) Run(event *beat.Event) (*beat.Event, error) { return event, nil } +func (d *addDockerMetadata) Close() error { + if d.cgroups != nil { + d.cgroups.StopJanitor() + } + d.watcher.Stop() + err := processors.Close(d.sourceProcessor) + if err != nil { + return errors.Wrap(err, "closing source processor of add_docker_metadata") + } + return nil +} + func (d *addDockerMetadata) String() string { return fmt.Sprintf("%v=[match_fields=[%v] match_pids=[%v]]", processorName, strings.Join(d.fields, ", "), strings.Join(d.pidFields, ", ")) diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go new file mode 100644 index 00000000000..91d3315a401 --- /dev/null +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go @@ -0,0 +1,120 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux darwin windows +// +build integration + +package add_docker_metadata + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/docker" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" + dockertest "github.com/elastic/beats/v7/libbeat/tests/docker" + "github.com/elastic/beats/v7/libbeat/tests/resources" +) + +func TestAddDockerMetadata(t *testing.T) { + goroutines := resources.NewGoroutinesChecker() + defer goroutines.Check(t) + + client, err := docker.NewClient(defaultConfig().Host, nil, nil) + require.NoError(t, err) + + // Docker clients can affect the goroutines checker because they keep + // idle keep-alive connections, so we explicitly close them. + // These idle connections in principle wouldn't represent leaks even if + // the client is not explicitly closed because they are eventually closed. + defer client.Close() + + // Start a container to have some data to enrich events + testClient, err := dockertest.NewClient() + require.NoError(t, err) + // Explicitly close client to don't affect goroutines checker + defer testClient.Close() + + image := "busybox" + cmd := []string{"sleep", "60"} + labels := map[string]string{"label": "foo"} + id, err := testClient.ContainerStart(image, cmd, labels) + require.NoError(t, err) + defer testClient.ContainerRemove(id) + + info, err := testClient.ContainerInspect(id) + require.NoError(t, err) + pid := info.State.Pid + + config, err := common.NewConfigFrom(map[string]interface{}{ + "match_fields": []string{"cid"}, + }) + watcherConstructor := newWatcherWith(client) + processor, err := buildDockerMetadataProcessor(logp.L(), config, watcherConstructor) + require.NoError(t, err) + + t.Run("match container by container id", func(t *testing.T) { + input := &beat.Event{Fields: common.MapStr{ + "cid": id, + }} + result, err := processor.Run(input) + require.NoError(t, err) + + resultLabels, _ := result.Fields.GetValue("container.labels") + expectedLabels := common.MapStr{"label": "foo"} + assert.Equal(t, expectedLabels, resultLabels) + assert.Equal(t, id, result.Fields["cid"]) + }) + + t.Run("match container by process id", func(t *testing.T) { + input := &beat.Event{Fields: common.MapStr{ + "cid": id, + "process.pid": pid, + }} + result, err := processor.Run(input) + require.NoError(t, err) + + resultLabels, _ := result.Fields.GetValue("container.labels") + expectedLabels := common.MapStr{"label": "foo"} + assert.Equal(t, expectedLabels, resultLabels) + assert.Equal(t, id, result.Fields["cid"]) + }) + + t.Run("don't enrich non existing container", func(t *testing.T) { + input := &beat.Event{Fields: common.MapStr{ + "cid": "notexists", + }} + result, err := processor.Run(input) + require.NoError(t, err) + assert.Equal(t, input.Fields, result.Fields) + }) + + err = processors.Close(processor) + require.NoError(t, err) +} + +func newWatcherWith(client docker.Client) docker.WatcherConstructor { + return func(log *logp.Logger, host string, tls *docker.TLSConfig, storeShortID bool) (docker.Watcher, error) { + return docker.NewWatcherWithClient(log, client, 60*time.Second, storeShortID) + } +} diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 2d8a5a9e970..f246d597e13 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -16,6 +16,7 @@ // under the License. // +build linux darwin windows +// +build !integration package add_docker_metadata diff --git a/libbeat/processors/add_kubernetes_metadata/cache.go b/libbeat/processors/add_kubernetes_metadata/cache.go index da7492b43a9..5de52efdc1b 100644 --- a/libbeat/processors/add_kubernetes_metadata/cache.go +++ b/libbeat/processors/add_kubernetes_metadata/cache.go @@ -31,6 +31,7 @@ type cache struct { timeout time.Duration deleted map[string]time.Time // key -> when should this obj be deleted metadata map[string]common.MapStr + done chan struct{} } func newCache(cleanupTimeout time.Duration) *cache { @@ -38,6 +39,7 @@ func newCache(cleanupTimeout time.Duration) *cache { timeout: cleanupTimeout, deleted: make(map[string]time.Time), metadata: make(map[string]common.MapStr), + done: make(chan struct{}), } go c.cleanup() return c @@ -67,15 +69,29 @@ func (c *cache) set(key string, data common.MapStr) { } func (c *cache) cleanup() { - ticker := time.Tick(timeout) - for now := range ticker { - c.Lock() - for k, t := range c.deleted { - if now.After(t) { - delete(c.deleted, k) - delete(c.metadata, k) + if timeout <= 0 { + return + } + + ticker := time.NewTicker(timeout) + defer ticker.Stop() + for { + select { + case <-c.done: + return + case now := <-ticker.C: + c.Lock() + for k, t := range c.deleted { + if now.After(t) { + delete(c.deleted, k) + delete(c.metadata, k) + } } + c.Unlock() } - c.Unlock() } } + +func (c *cache) stop() { + close(c.done) +} diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index 2a5f4d2faed..535eb1187ea 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -242,6 +242,16 @@ func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) { return event, nil } +func (k *kubernetesAnnotator) Close() error { + if k.watcher != nil { + k.watcher.Stop() + } + if k.cache != nil { + k.cache.stop() + } + return nil +} + func (k *kubernetesAnnotator) addPod(pod *kubernetes.Pod) { metadata := k.indexers.GetMetadata(pod) for _, m := range metadata { diff --git a/libbeat/processors/add_process_metadata/add_process_metadata.go b/libbeat/processors/add_process_metadata/add_process_metadata.go index c41ca9a73d6..01e2cf1e9fe 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata.go @@ -55,11 +55,12 @@ var ( ) type addProcessMetadata struct { - config config - provider processMetadataProvider - cidProvider cidProvider - log *logp.Logger - mappings common.MapStr + config config + provider processMetadataProvider + cgroupsCache *common.Cache + cidProvider cidProvider + log *logp.Logger + mappings common.MapStr } type processMetadata struct { @@ -81,16 +82,22 @@ type cidProvider interface { } func init() { - processors.RegisterPlugin(processorName, New) + processors.RegisterPlugin(processorName, NewWithCache) jsprocessor.RegisterPlugin("AddProcessMetadata", New) } // New constructs a new add_process_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { - return newProcessMetadataProcessorWithProvider(cfg, &procCache) + return newProcessMetadataProcessorWithProvider(cfg, &procCache, false) } -func newProcessMetadataProcessorWithProvider(cfg *common.Config, provider processMetadataProvider) (proc processors.Processor, err error) { +// NewWithCache construct a new add_process_metadata processor with cache for container IDs. +// Resulting processor implements `Close()` to release the cache resources. +func NewWithCache(cfg *common.Config) (processors.Processor, error) { + return newProcessMetadataProcessorWithProvider(cfg, &procCache, true) +} + +func newProcessMetadataProcessorWithProvider(cfg *common.Config, provider processMetadataProvider, withCache bool) (proc processors.Processor, err error) { // Logging (each processor instance has a unique ID). var ( id = int(instanceID.Inc()) @@ -118,21 +125,25 @@ func newProcessMetadataProcessorWithProvider(cfg *common.Config, provider proces } // don't use cgroup.ProcessCgroupPaths to save it from doing the work when container id disabled if ok := containsValue(mappings, "container.id"); ok { - if config.CgroupCacheExpireTime != 0 { + if withCache && config.CgroupCacheExpireTime != 0 { p.log.Debug("Initializing cgroup cache") evictionListener := func(k common.Key, v common.Value) { p.log.Debugf("Evicted cached cgroups for PID=%v", k) } - cgroupsCache := common.NewCacheWithRemovalListener(config.CgroupCacheExpireTime, 100, evictionListener) - cgroupsCache.StartJanitor(config.CgroupCacheExpireTime) - p.cidProvider = newCidProvider(config.HostPath, config.CgroupPrefixes, config.CgroupRegex, processCgroupPaths, cgroupsCache) + p.cgroupsCache = common.NewCacheWithRemovalListener(config.CgroupCacheExpireTime, 100, evictionListener) + p.cgroupsCache.StartJanitor(config.CgroupCacheExpireTime) + p.cidProvider = newCidProvider(config.HostPath, config.CgroupPrefixes, config.CgroupRegex, processCgroupPaths, p.cgroupsCache) } else { p.cidProvider = newCidProvider(config.HostPath, config.CgroupPrefixes, config.CgroupRegex, processCgroupPaths, nil) } } + if withCache { + return &addProcessMetadataCloser{p}, nil + } + return &p, nil } @@ -253,6 +264,17 @@ func (p *addProcessMetadata) getContainerID(pid int) (string, error) { return cid, nil } +type addProcessMetadataCloser struct { + addProcessMetadata +} + +func (p *addProcessMetadataCloser) Close() error { + if p.addProcessMetadata.cgroupsCache != nil { + p.addProcessMetadata.cgroupsCache.StopJanitor() + } + return nil +} + // String returns the processor representation formatted as a string func (p *addProcessMetadata) String() string { return fmt.Sprintf("%v=[match_pids=%v, mappings=%v, ignore_missing=%v, overwrite_fields=%v, restricted_fields=%v, host_path=%v, cgroup_prefixes=%v]", diff --git a/libbeat/processors/add_process_metadata/add_process_metadata_test.go b/libbeat/processors/add_process_metadata/add_process_metadata_test.go index f9b4aaa681c..493034098cf 100644 --- a/libbeat/processors/add_process_metadata/add_process_metadata_test.go +++ b/libbeat/processors/add_process_metadata/add_process_metadata_test.go @@ -651,7 +651,7 @@ func TestAddProcessMetadata(t *testing.T) { t.Fatal(err) } - proc, err := newProcessMetadataProcessorWithProvider(config, testProcs) + proc, err := newProcessMetadataProcessorWithProvider(config, testProcs, true) if test.initErr == nil { if err != nil { t.Fatal(err) diff --git a/libbeat/processors/dns/config.go b/libbeat/processors/dns/config.go index 8b0d6c9bd97..a2c2a98955e 100644 --- a/libbeat/processors/dns/config.go +++ b/libbeat/processors/dns/config.go @@ -31,7 +31,7 @@ import ( type Config struct { CacheConfig `config:",inline"` Nameservers []string `config:"nameservers"` // Required on Windows. /etc/resolv.conf is used if none are given. - Timeout time.Duration `conifg:"timeout"` // Per request timeout (with 2 nameservers the total timeout would be 2x). + Timeout time.Duration `config:"timeout"` // Per request timeout (with 2 nameservers the total timeout would be 2x). Type string `config:"type" validate:"required"` // Reverse is the only supported type currently. Action FieldAction `config:"action"` // Append or replace (defaults to append) when target exists. TagOnFailure []string `config:"tag_on_failure"` // Tags to append when a failure occurs. diff --git a/libbeat/processors/processor.go b/libbeat/processors/processor.go index c5002b7cebd..0e772a524da 100644 --- a/libbeat/processors/processor.go +++ b/libbeat/processors/processor.go @@ -20,6 +20,7 @@ package processors import ( "strings" + "github.com/joeshaw/multierror" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/beat" @@ -35,11 +36,29 @@ type Processors struct { log *logp.Logger } +// Processor is the interface that all processors must implement type Processor interface { Run(event *beat.Event) (*beat.Event, error) String() string } +// Closer defines the interface for processors that should be closed after using +// them. +// Close() is not part of the Processor interface because implementing this method +// is also a way to indicate that the processor keeps some resource that needs to +// be released or orderly closed. +type Closer interface { + Close() error +} + +// Close closes a processor if it implements the Closer interface +func Close(p Processor) error { + if closer, ok := p.(Closer); ok { + return closer.Close() + } + return nil +} + // NewList creates a new empty processor list. // Additional processors can be added to the List field. func NewList(log *logp.Logger) *Processors { @@ -153,6 +172,17 @@ func (procs *Processors) All() []beat.Processor { return ret } +func (procs *Processors) Close() error { + var errs multierror.Errors + for _, p := range procs.List { + err := Close(p) + if err != nil { + errs = append(errs, err) + } + } + return errs.Err() +} + // Run executes the all processors serially and returns the event and possibly // an error. If the event has been dropped (canceled) by a processor in the // list then a nil event is returned. diff --git a/libbeat/processors/script/javascript/module/processor/chain.go b/libbeat/processors/script/javascript/module/processor/chain.go index e58aac29372..9ef3da7859e 100644 --- a/libbeat/processors/script/javascript/module/processor/chain.go +++ b/libbeat/processors/script/javascript/module/processor/chain.go @@ -151,6 +151,17 @@ func newNativeProcessor(constructor processors.Constructor, call gojaCall) (proc if err != nil { return nil, err } + + if closer, ok := p.(processors.Closer); ok { + closer.Close() + // Script processor doesn't support releasing resources of stateful processors, + // what can lead to leaks, so prevent use of these processors. They shouldn't + // be registered. If this error happens, a processor that needs to be closed is + // being registered, this should be avoided. + // See https://github.com/elastic/beats/pull/16349 + return nil, errors.Errorf("stateful processor cannot be used in script processor, this is probably a bug: %s", p) + } + return &nativeProcessor{p}, nil } diff --git a/libbeat/processors/script/javascript/module/processor/processor_test.go b/libbeat/processors/script/javascript/module/processor/processor_test.go index 6ea66f409ff..9e958ee7035 100644 --- a/libbeat/processors/script/javascript/module/processor/processor_test.go +++ b/libbeat/processors/script/javascript/module/processor/processor_test.go @@ -23,6 +23,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" @@ -35,6 +36,7 @@ import ( func init() { RegisterPlugin("Mock", newMock) + RegisterPlugin("MockWithCloser", newMockWithCloser) } func testEvent() *beat.Event { @@ -67,14 +69,10 @@ function process(evt) { logp.TestingSetup() p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) checkEvent(t, evt, "added", "new_value") } @@ -107,14 +105,10 @@ function process(evt) { logp.TestingSetup() p, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) evt, err := p.Run(testEvent()) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) // checking if hello world is added to the event in different languages checkEvent(t, evt, "helló", "világ") @@ -123,6 +117,22 @@ function process(evt) { checkEvent(t, evt, "hallo", "Welt") } +func TestProcessorWithCloser(t *testing.T) { + const script = ` +var processor = require('processor'); + +var processorWithCloser = new processor.MockWithCloser().Build() + +function process(evt) { + processorWithCloser.Run(evt); +} +` + + logp.TestingSetup() + _, err := javascript.NewFromConfig(javascript.Config{Source: script}, nil) + require.Error(t, err, "processor that implements Closer() shouldn't be allowed") +} + func checkEvent(t *testing.T, evt *beat.Event, key, value string) { s, err := evt.GetValue(key) assert.NoError(t, err) @@ -162,3 +172,23 @@ func (m *mockProcessor) String() string { s, _ := json.Marshal(m.fields) return fmt.Sprintf("mock=%s", s) } + +type mockProcessorWithCloser struct{} + +func newMockWithCloser(c *common.Config) (processors.Processor, error) { + return &mockProcessorWithCloser{}, nil +} + +func (m *mockProcessorWithCloser) Run(event *beat.Event) (*beat.Event, error) { + // Nothing to do, we only want this struct to implement processors.Closer + return event, nil +} + +func (m *mockProcessorWithCloser) Close() error { + // Nothing to do, we only want this struct to implement processors.Closer + return nil +} + +func (m *mockProcessorWithCloser) String() string { + return "mockWithCloser" +} diff --git a/libbeat/processors/timestamp/docs/timestamp.asciidoc b/libbeat/processors/timestamp/docs/timestamp.asciidoc index 83421c584a7..eb5ba628995 100644 --- a/libbeat/processors/timestamp/docs/timestamp.asciidoc +++ b/libbeat/processors/timestamp/docs/timestamp.asciidoc @@ -50,7 +50,7 @@ If a layout does not contain a year then the current year in the specified Here is an example that parses the `start_time` field and writes the result to the `@timestamp` field then deletes the `start_time` field. When the -processor is loaded it will immediately validate that the two `test` timestamps +processor is loaded, it will immediately validate that the two `test` timestamps parse with this configuration. [source,yaml] @@ -61,9 +61,11 @@ processors: layouts: - '2006-01-02T15:04:05Z' - '2006-01-02T15:04:05.999Z' + - '2006-01-02T15:04:05.999-07:00' test: - '2019-06-22T16:33:51Z' - '2019-11-18T04:59:51.123Z' + - '2020-08-03T07:10:20.123456+02:00' - drop_fields: fields: [start_time] ---- diff --git a/libbeat/publisher/pipeline/client.go b/libbeat/publisher/pipeline/client.go index 2ce792ed887..edb5a3f1eb3 100644 --- a/libbeat/publisher/pipeline/client.go +++ b/libbeat/publisher/pipeline/client.go @@ -24,6 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/beats/v7/libbeat/publisher/queue" ) @@ -164,6 +165,15 @@ func (c *client) Close() error { log.Debug("client: unlink from queue") c.unlink() log.Debug("client: done unlink") + + if c.processors != nil { + log.Debug("client: closing processors") + err := processors.Close(c.processors) + if err != nil { + log.Errorf("client: error closing processors: %v", err) + } + log.Debug("client: done closing processors") + } }) return nil } diff --git a/libbeat/publisher/pipeline/controller_test.go b/libbeat/publisher/pipeline/controller_test.go index fba98018894..1e400476a30 100644 --- a/libbeat/publisher/pipeline/controller_test.go +++ b/libbeat/publisher/pipeline/controller_test.go @@ -31,7 +31,8 @@ import ( "github.com/elastic/beats/v7/libbeat/publisher" "github.com/elastic/beats/v7/libbeat/publisher/queue" "github.com/elastic/beats/v7/libbeat/publisher/queue/memqueue" - "github.com/elastic/beats/v7/libbeat/tests/resources" + + //"github.com/elastic/beats/v7/libbeat/tests/resources" "github.com/stretchr/testify/require" ) @@ -46,8 +47,9 @@ func TestOutputReload(t *testing.T) { t.Run(name, func(t *testing.T) { testutil.SeedPRNG(t) - goroutines := resources.NewGoroutinesChecker() - defer goroutines.Check(t) + // Flaky check: https://github.com/elastic/beats/issues/21656 + //goroutines := resources.NewGoroutinesChecker() + //defer goroutines.Check(t) err := quick.Check(func(q uint) bool { numEventsToPublish := 15000 + (q % 500) // 15000 to 19999 diff --git a/libbeat/publisher/pipeline/output_test.go b/libbeat/publisher/pipeline/output_test.go index 5be34fa9436..7bde6e137ab 100644 --- a/libbeat/publisher/pipeline/output_test.go +++ b/libbeat/publisher/pipeline/output_test.go @@ -95,6 +95,8 @@ func TestMakeClientWorker(t *testing.T) { } func TestReplaceClientWorker(t *testing.T) { + t.Skip("Flaky test: https://github.com/elastic/beats/issues/17965") + tests := map[string]func(mockPublishFn) outputs.Client{ "client": newMockClient, "network_client": newMockNetworkClient, diff --git a/libbeat/publisher/processing/default.go b/libbeat/publisher/processing/default.go index b2a65642e17..f9eab88fe48 100644 --- a/libbeat/publisher/processing/default.go +++ b/libbeat/publisher/processing/default.go @@ -338,7 +338,10 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, } // setup 8: pipeline processors list - processors.add(b.processors) + if b.processors != nil { + // Add the global pipeline as a function processor, so clients cannot close it + processors.add(newProcessor(b.processors.title, b.processors.Run)) + } // setup 9: time series metadata if b.timeSeries { @@ -358,6 +361,13 @@ func (b *builder) Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, return processors, nil } +func (b *builder) Close() error { + if b.processors != nil { + return b.processors.Close() + } + return nil +} + func makeClientProcessors( log *logp.Logger, cfg beat.ProcessingConfig, diff --git a/libbeat/publisher/processing/default_test.go b/libbeat/publisher/processing/default_test.go index 56dfa75bdd2..637b38cf44d 100644 --- a/libbeat/publisher/processing/default_test.go +++ b/libbeat/publisher/processing/default_test.go @@ -29,6 +29,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/actions" "github.com/elastic/ecs/code/go/ecs" ) @@ -317,6 +318,9 @@ func TestNormalization(t *testing.T) { fields.DeepUpdate(test.mod) assert.Equal(t, test.want, actual.Fields) + + err = s.Close() + require.NoError(t, err) }) } } @@ -331,6 +335,9 @@ func TestAlwaysDrop(t *testing.T) { actual, err := prog.Run(&beat.Event{}) require.NoError(t, err) assert.Nil(t, actual) + + err = s.Close() + require.NoError(t, err) } func TestDynamicFields(t *testing.T) { @@ -351,6 +358,52 @@ func TestDynamicFields(t *testing.T) { actual, err = prog.Run(&beat.Event{Fields: common.MapStr{"hello": "world"}}) require.NoError(t, err) assert.Equal(t, common.MapStr{"hello": "world", "dyn": "field"}, actual.Fields) + + err = factory.Close() + require.NoError(t, err) +} + +func TestProcessingClose(t *testing.T) { + factory, err := MakeDefaultSupport(true)(beat.Info{}, logp.L(), common.NewConfig()) + require.NoError(t, err) + + // Inject a processor in the builder that we can check if has been closed. + factoryProcessor := &processorWithClose{} + b := factory.(*builder) + if b.processors == nil { + b.processors = newGroup("global", logp.L()) + } + b.processors.add(factoryProcessor) + + clientProcessor := &processorWithClose{} + g := newGroup("test", logp.L()) + g.add(clientProcessor) + + prog, err := factory.Create(beat.ProcessingConfig{ + Processor: g, + }, false) + require.NoError(t, err) + + // Check that both processors are called + assert.False(t, factoryProcessor.called) + assert.False(t, clientProcessor.called) + _, err = prog.Run(&beat.Event{Fields: common.MapStr{"hello": "world"}}) + require.NoError(t, err) + assert.True(t, factoryProcessor.called) + assert.True(t, clientProcessor.called) + + // Check that closing the client processing pipeline doesn't close the global pipeline + assert.False(t, factoryProcessor.closed) + assert.False(t, clientProcessor.closed) + err = processors.Close(prog) + require.NoError(t, err) + assert.False(t, factoryProcessor.closed) + assert.True(t, clientProcessor.closed) + + // Check that closing the factory closes the processor in the global pipeline + err = factory.Close() + require.NoError(t, err) + assert.True(t, factoryProcessor.closed) } func fromJSON(in string) common.MapStr { @@ -361,3 +414,22 @@ func fromJSON(in string) common.MapStr { } return tmp } + +type processorWithClose struct { + closed bool + called bool +} + +func (p *processorWithClose) Run(e *beat.Event) (*beat.Event, error) { + p.called = true + return e, nil +} + +func (p *processorWithClose) Close() error { + p.closed = true + return nil +} + +func (p *processorWithClose) String() string { + return "processorWithClose" +} diff --git a/libbeat/publisher/processing/processing.go b/libbeat/publisher/processing/processing.go index 4f615fb1422..88feb62c7b2 100644 --- a/libbeat/publisher/processing/processing.go +++ b/libbeat/publisher/processing/processing.go @@ -33,6 +33,8 @@ type SupportFactory func(info beat.Info, log *logp.Logger, cfg *common.Config) ( // will merge the global and local configurations into a common event // processor. // If `drop` is set, then the processor generated must always drop all events. +// A Supporter needs to be closed with `Close()` to release its global resources. type Supporter interface { Create(cfg beat.ProcessingConfig, drop bool) (beat.Processor, error) + Close() error } diff --git a/libbeat/publisher/processing/processors.go b/libbeat/publisher/processing/processors.go index e994eef48cc..3a400d36dad 100644 --- a/libbeat/publisher/processing/processors.go +++ b/libbeat/publisher/processing/processors.go @@ -22,6 +22,8 @@ import ( "strings" "sync" + "github.com/joeshaw/multierror" + "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" @@ -77,6 +79,20 @@ func (p *group) add(processor processors.Processor) { } } +func (p *group) Close() error { + if p == nil { + return nil + } + var errs multierror.Errors + for _, processor := range p.list { + err := processors.Close(processor) + if err != nil { + errs = append(errs, err) + } + } + return errs.Err() +} + func (p *group) String() string { var s []string for _, p := range p.list { diff --git a/libbeat/publisher/queue/diskqueue/config.go b/libbeat/publisher/queue/diskqueue/config.go index 6a165a665db..b8ef456d03d 100644 --- a/libbeat/publisher/queue/diskqueue/config.go +++ b/libbeat/publisher/queue/diskqueue/config.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "path/filepath" + "time" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/cfgtype" @@ -61,16 +62,26 @@ type Settings struct { // A listener that should be sent ACKs when an event is successfully // written to disk. WriteToDiskListener queue.ACKListener + + // RetryInterval specifies how long to wait before retrying a fatal error + // writing to disk. If MaxRetryInterval is nonzero, subsequent retries will + // use exponential backoff up to the specified limit. + RetryInterval time.Duration + MaxRetryInterval time.Duration } // userConfig holds the parameters for a disk queue that are configurable // by the end user in the beats yml file. type userConfig struct { - Path string `config:"path"` - MaxSize cfgtype.ByteSize `config:"max_size" validate:"required"` - SegmentSize *cfgtype.ByteSize `config:"segment_size"` - ReadAheadLimit *int `config:"read_ahead"` - WriteAheadLimit *int `config:"write_ahead"` + Path string `config:"path"` + MaxSize cfgtype.ByteSize `config:"max_size" validate:"required"` + SegmentSize *cfgtype.ByteSize `config:"segment_size"` + + ReadAheadLimit *int `config:"read_ahead"` + WriteAheadLimit *int `config:"write_ahead"` + + RetryInterval *time.Duration `config:"retry_interval" validate:"positive"` + MaxRetryInterval *time.Duration `config:"max_retry_interval" validate:"positive"` } func (c *userConfig) Validate() error { @@ -96,6 +107,13 @@ func (c *userConfig) Validate() error { "Disk queue segment_size (%d) cannot be less than 1MB", *c.SegmentSize) } + if c.RetryInterval != nil && c.MaxRetryInterval != nil && + *c.MaxRetryInterval < *c.RetryInterval { + return fmt.Errorf( + "Disk queue max_retry_interval (%v) can't be less than retry_interval (%v)", + *c.MaxRetryInterval, *c.RetryInterval) + } + return nil } @@ -108,6 +126,9 @@ func DefaultSettings() Settings { ReadAheadLimit: 512, WriteAheadLimit: 2048, + + RetryInterval: 1 * time.Second, + MaxRetryInterval: 30 * time.Second, } } @@ -137,6 +158,13 @@ func SettingsForUserConfig(config *common.Config) (Settings, error) { settings.WriteAheadLimit = *userConfig.WriteAheadLimit } + if userConfig.RetryInterval != nil { + settings.RetryInterval = *userConfig.RetryInterval + } + if userConfig.MaxRetryInterval != nil { + settings.MaxRetryInterval = *userConfig.RetryInterval + } + return settings, nil } @@ -164,3 +192,17 @@ func (settings Settings) segmentPath(segmentID segmentID) string { func (settings Settings) maxSegmentOffset() segmentOffset { return segmentOffset(settings.MaxSegmentSize - segmentHeaderSize) } + +// Given a retry interval, nextRetryInterval returns the next higher level +// of backoff. +func (settings Settings) nextRetryInterval( + currentInterval time.Duration, +) time.Duration { + if settings.MaxRetryInterval > 0 { + currentInterval *= 2 + if currentInterval > settings.MaxRetryInterval { + currentInterval = settings.MaxRetryInterval + } + } + return currentInterval +} diff --git a/libbeat/publisher/queue/diskqueue/deleter_loop.go b/libbeat/publisher/queue/diskqueue/deleter_loop.go index 4e685285948..3948d200cbc 100644 --- a/libbeat/publisher/queue/diskqueue/deleter_loop.go +++ b/libbeat/publisher/queue/diskqueue/deleter_loop.go @@ -57,6 +57,7 @@ func newDeleterLoop(settings Settings) *deleterLoop { } func (dl *deleterLoop) run() { + currentRetryInterval := dl.settings.RetryInterval for { request, ok := <-dl.requestChan if !ok { @@ -87,10 +88,14 @@ func (dl *deleterLoop) run() { // The delay can be interrupted if the request channel is closed, // indicating queue shutdown. select { - // TODO: make the retry interval configurable. - case <-time.After(time.Second): + case <-time.After(currentRetryInterval): case <-dl.requestChan: } + currentRetryInterval = + dl.settings.nextRetryInterval(currentRetryInterval) + } else { + // If we made progress, reset the retry interval. + currentRetryInterval = dl.settings.RetryInterval } dl.responseChan <- deleterLoopResponse{ results: results, diff --git a/libbeat/publisher/queue/diskqueue/segments.go b/libbeat/publisher/queue/diskqueue/segments.go index 5ce0dc49962..617b089110e 100644 --- a/libbeat/publisher/queue/diskqueue/segments.go +++ b/libbeat/publisher/queue/diskqueue/segments.go @@ -207,10 +207,15 @@ func (segment *queueSegment) getWriter( // retry callback returns true. This is used for timed retries when // creating a queue segment from the writer loop. func (segment *queueSegment) getWriterWithRetry( - queueSettings Settings, retry func(error) bool, + queueSettings Settings, retry func(err error, firstTime bool) bool, ) (*os.File, error) { + firstTime := true file, err := segment.getWriter(queueSettings) - for err != nil && retry(err) { + for err != nil && retry(err, firstTime) { + // Set firstTime to false so the retry callback can perform backoff + // etc if needed. + firstTime = false + // Try again file, err = segment.getWriter(queueSettings) } diff --git a/libbeat/publisher/queue/diskqueue/util.go b/libbeat/publisher/queue/diskqueue/util.go index 60c529a9992..c54a26154e8 100644 --- a/libbeat/publisher/queue/diskqueue/util.go +++ b/libbeat/publisher/queue/diskqueue/util.go @@ -69,16 +69,32 @@ func writeErrorIsRetriable(err error) bool { // "wrapped" field in-place as long as it isn't captured by the callback. type callbackRetryWriter struct { wrapped io.Writer - retry func(error) bool + + // The retry callback is called with the error that was produced and whether + // this is the first (subsequent) error arising from this particular + // write call. + retry func(err error, firstTime bool) bool } func (w callbackRetryWriter) Write(p []byte) (int, error) { + // firstTime tracks whether the current error is the first subsequent error + // being passed to the retry callback. This is so that the callback can + // reset its internal counters in case it is using exponential backoff or + // a retry limit. + firstTime := true bytesWritten := 0 writer := w.wrapped n, err := writer.Write(p) for n < len(p) { - if err != nil && !w.retry(err) { - return bytesWritten + n, err + if err != nil { + shouldRetry := w.retry(err, firstTime) + firstTime = false + if !shouldRetry { + return bytesWritten + n, err + } + } else { + // If we made progress without an error, reset firstTime. + firstTime = true } // Advance p and try again. bytesWritten += n diff --git a/libbeat/publisher/queue/diskqueue/writer_loop.go b/libbeat/publisher/queue/diskqueue/writer_loop.go index b42e4573cab..ff1ff97616a 100644 --- a/libbeat/publisher/queue/diskqueue/writer_loop.go +++ b/libbeat/publisher/queue/diskqueue/writer_loop.go @@ -82,6 +82,8 @@ type writerLoop struct { // The file handle corresponding to currentSegment. When currentSegment // changes, this handle is closed and a new one is created. outputFile *os.File + + currentRetryInterval time.Duration } func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { @@ -91,6 +93,8 @@ func newWriterLoop(logger *logp.Logger, settings Settings) *writerLoop { requestChan: make(chan writerLoopRequest, 1), responseChan: make(chan writerLoopResponse), + + currentRetryInterval: settings.RetryInterval, } } @@ -215,14 +219,31 @@ outerLoop: return append(bytesWritten, curBytesWritten) } -// retryCallback is called (by way of retryCallbackWriter) when there is +func (wl *writerLoop) applyRetryBackoff() { + wl.currentRetryInterval = + wl.settings.nextRetryInterval(wl.currentRetryInterval) +} + +func (wl *writerLoop) resetRetryBackoff() { + wl.currentRetryInterval = wl.settings.RetryInterval +} + +// retryCallback is called (by way of callbackRetryWriter) when there is // an error writing to a segment file. It pauses for a configurable // interval and returns true if the operation should be retried (which // it always should, unless the queue is being closed). -func (wl *writerLoop) retryCallback(err error) bool { +func (wl *writerLoop) retryCallback(err error, firstTime bool) bool { + if firstTime { + // Reset any exponential backoff in the retry interval. + wl.resetRetryBackoff() + } if writeErrorIsRetriable(err) { return true } + // If this error isn't immediately retriable, increase the exponential + // backoff afterwards. + defer wl.applyRetryBackoff() + // If the error is not immediately retriable, log the error // and wait for the retry interval before trying again, but // abort if the queue is closed (indicated by the request channel @@ -230,8 +251,7 @@ func (wl *writerLoop) retryCallback(err error) bool { wl.logger.Errorf("Writing to segment %v: %v", wl.currentSegment.id, err) select { - case <-time.After(time.Second): - // TODO: use a configurable interval here + case <-time.After(wl.currentRetryInterval): return true case <-wl.requestChan: return false diff --git a/libbeat/scripts/generate_fields_docs.py b/libbeat/scripts/generate_fields_docs.py index 89a80d83d27..8e0a7c77c32 100644 --- a/libbeat/scripts/generate_fields_docs.py +++ b/libbeat/scripts/generate_fields_docs.py @@ -1,6 +1,7 @@ import argparse from collections import OrderedDict import os +import re import yaml @@ -20,11 +21,24 @@ def document_fields(output, section, sections, path): output.write("[float]\n") if "description" in section: - if "anchor" in section: + if "anchor" in section and section["name"] == "ECS": output.write("== {} fields\n\n".format(section["name"])) + output.write(""" +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. +""") + elif "anchor" in section: + output.write("== {} fields\n\n".format(section["name"])) + output.write("{}\n\n".format(section["description"])) else: output.write("=== {}\n\n".format(section["name"])) - output.write("{}\n\n".format(section["description"])) + output.write("{}\n\n".format(section["description"])) if "fields" not in section or not section["fields"]: return @@ -70,6 +84,14 @@ def document_field(output, field, field_path): if "path" in field: output.write("alias to: {}\n\n".format(field["path"])) + # For Apm-Server docs only + # Assign an ECS badge for fields containing "overwrite" + if beat_title == "Apm-Server": + if "overwrite" in field: + # And it's not a Kubernetes field + if re.match("^kubernetes.*", field["field_path"]) is None: + output.write("{yes-icon} {ecs-ref}[ECS] field.\n\n") + if "index" in field: if not field["index"]: output.write("{}\n\n".format("Field is not indexed.")) diff --git a/libbeat/tests/docker/docker.go b/libbeat/tests/docker/docker.go index b81ffa285ea..888347c5cc7 100644 --- a/libbeat/tests/docker/docker.go +++ b/libbeat/tests/docker/docker.go @@ -77,8 +77,28 @@ func (c Client) ContainerWait(ID string) error { return nil } +// ContainerInspect recovers information of the container +func (c Client) ContainerInspect(ID string) (types.ContainerJSON, error) { + ctx := context.Background() + return c.cli.ContainerInspect(ctx, ID) +} + // ContainerKill kills the given container func (c Client) ContainerKill(ID string) error { ctx := context.Background() return c.cli.ContainerKill(ctx, ID, "KILL") } + +// ContainerRemove kills and removed the given container +func (c Client) ContainerRemove(ID string) error { + ctx := context.Background() + return c.cli.ContainerRemove(ctx, ID, types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + }) +} + +// Close closes the underlying client +func (c *Client) Close() error { + return c.cli.Close() +} diff --git a/libbeat/tests/system/beat/common_tests.py b/libbeat/tests/system/beat/common_tests.py index c9cdbc52cc0..920ee35e72e 100644 --- a/libbeat/tests/system/beat/common_tests.py +++ b/libbeat/tests/system/beat/common_tests.py @@ -4,6 +4,12 @@ from beat.beat import INTEGRATION_TESTS +# Fail if the exported index pattern is larger than 10MiB +# This is to avoid problems with Kibana when the payload +# of the request to install the index pattern exceeds the +# default limit. +index_pattern_size_limit = 10 * 1024 * 1024 + class TestExportsMixin: @@ -56,7 +62,7 @@ def test_export_index_pattern(self): js = json.loads(output) assert "objects" in js size = len(output.encode('utf-8')) - assert size < 1024 * 1024, "Kibana index pattern must be less than 1MiB " \ + assert size < index_pattern_size_limit, "Kibana index pattern must be less than 10MiB " \ "to keep the Beat setup request size below " \ "Kibana's server.maxPayloadBytes." @@ -68,7 +74,7 @@ def test_export_index_pattern_migration(self): js = json.loads(output) assert "objects" in js size = len(output.encode('utf-8')) - assert size < 1024 * 1024, "Kibana index pattern must be less than 1MiB " \ + assert size < index_pattern_size_limit, "Kibana index pattern must be less than 10MiB " \ "to keep the Beat setup request size below " \ "Kibana's server.maxPayloadBytes." diff --git a/libbeat/tests/system/test_cmd_completion.py b/libbeat/tests/system/test_cmd_completion.py index 2e523c0831d..a06259a5046 100644 --- a/libbeat/tests/system/test_cmd_completion.py +++ b/libbeat/tests/system/test_cmd_completion.py @@ -17,7 +17,7 @@ def test_bash_completion(self): def test_zsh_completion(self): exit_code = self.run_beat(extra_args=["completion", "zsh"]) assert exit_code == 0 - assert self.log_contains("#compdef mockbeat") + assert self.log_contains("#compdef _mockbeat mockbeat") def test_unknown_completion(self): exit_code = self.run_beat(extra_args=["completion", "awesomeshell"]) diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 09f1a49b9e2..9860919d006 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -40,3 +40,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test metricbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index ae34419db2e..a193d630564 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4665,6 +4665,16 @@ type: keyword The subscription ID +type: keyword + +-- + +*`azure.application_id`*:: ++ +-- +The application ID + + type: keyword -- @@ -4696,17 +4706,44 @@ application insights -*`azure.app_insights.application_id`*:: +*`azure.app_insights.start_date`*:: + -- -The application ID +The start date -type: keyword +type: date -- -*`azure.app_insights.start_date`*:: +*`azure.app_insights.end_date`*:: ++ +-- +The end date + + +type: date + +-- + +*`azure.app_insights.metrics.*.*`*:: ++ +-- +The metrics + + +type: object + +-- + +[float] +=== app_state + +application state + + + +*`azure.app_state.start_date`*:: + -- The start date @@ -4716,7 +4753,7 @@ type: date -- -*`azure.app_insights.end_date`*:: +*`azure.app_state.end_date`*:: + -- The end date @@ -4726,13 +4763,183 @@ type: date -- -*`azure.app_insights.metrics.*.*`*:: +*`azure.app_state.requests_count.sum`*:: + -- -The metrics +Request count -type: object +type: float + +-- + +*`azure.app_state.requests_failed.sum`*:: ++ +-- +Request failed count + + +type: float + +-- + +*`azure.app_state.users_count.unique`*:: ++ +-- +User count + + +type: float + +-- + +*`azure.app_state.sessions_count.unique`*:: ++ +-- +Session count + + +type: float + +-- + +*`azure.app_state.users_authenticated.unique`*:: ++ +-- +Authenticated users count + + +type: float + +-- + +*`azure.app_state.browser_timings_network_duration.avg`*:: ++ +-- +Browser timings network duration + + +type: float + +-- + +*`azure.app_state.browser_timings_send_duration.avg`*:: ++ +-- +Browser timings send duration + + +type: float + +-- + +*`azure.app_state.browser_timings_receive_uration.avg`*:: ++ +-- +Browser timings receive duration + + +type: float + +-- + +*`azure.app_state.browser_timings_processing_duration.avg`*:: ++ +-- +Browser timings processing duration + + +type: float + +-- + +*`azure.app_state.browser_timings_total_duration.avg`*:: ++ +-- +Browser timings total duration + + +type: float + +-- + +*`azure.app_state.exceptions_count.sum`*:: ++ +-- +Exception count + + +type: float + +-- + +*`azure.app_state.exceptions_browser.sum`*:: ++ +-- +Exception count at browser level + + +type: float + +-- + +*`azure.app_state.exceptions_server.sum`*:: ++ +-- +Exception count at server level + + +type: float + +-- + +*`azure.app_state.performance_counters_memory_available_bytes.avg`*:: ++ +-- +Performance counters memory available bytes + + +type: float + +-- + +*`azure.app_state.performance_counters_process_private_bytes.avg`*:: ++ +-- +Performance counters process private bytes + + +type: float + +-- + +*`azure.app_state.performance_counters_process_cpu_percentage_total.avg`*:: ++ +-- +Performance counters process cpu percentage total + + +type: float + +-- + +*`azure.app_state.performance_counters_process_cpu_percentage.avg`*:: ++ +-- +Performance counters process cpu percentage + + +type: float + +-- + +*`azure.app_state.performance_counters_processiobytes_per_second.avg`*:: ++ +-- +Performance counters process IO bytes per second + + +type: float -- @@ -9164,8 +9371,15 @@ Stats collected from Dropwizard. [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + @@ -24656,16 +24870,6 @@ type: long -- -*`kafka.partition.partition.isr`*:: -+ --- -List of isr ids. - - -type: keyword - --- - *`kafka.partition.partition.replica`*:: + -- @@ -28953,6 +29157,7 @@ linux module [float] === linux +linux system metrics @@ -29050,6 +29255,147 @@ type: long -- +[float] +=== iostat + +iostat + + + +*`linux.iostat.read.request.merges_per_sec`*:: ++ +-- +The number of read requests merged per second that were queued to the device. + + +type: float + +-- + +*`linux.iostat.write.request.merges_per_sec`*:: ++ +-- +The number of write requests merged per second that were queued to the device. + + +type: float + +-- + +*`linux.iostat.read.request.per_sec`*:: ++ +-- +The number of read requests that were issued to the device per second + + +type: float + +-- + +*`linux.iostat.write.request.per_sec`*:: ++ +-- +The number of write requests that were issued to the device per second + + +type: float + +-- + +*`linux.iostat.read.per_sec.bytes`*:: ++ +-- +The number of Bytes read from the device per second. + + +type: float + +format: bytes + +-- + +*`linux.iostat.read.await`*:: ++ +-- +The average time spent for read requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.write.per_sec.bytes`*:: ++ +-- +The number of Bytes write from the device per second. + + +type: float + +format: bytes + +-- + +*`linux.iostat.write.await`*:: ++ +-- +The average time spent for write requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.request.avg_size`*:: ++ +-- +The average size (in bytes) of the requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.queue.avg_size`*:: ++ +-- +The average queue length of the requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.await`*:: ++ +-- +The average time spent for requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.service_time`*:: ++ +-- +The average service time (in milliseconds) for I/O requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.busy`*:: ++ +-- +Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + + +type: float + +-- + [float] === ksm @@ -29124,6 +29470,186 @@ type: long -- +[float] +=== memory + +Linux memory data + + + +[float] +=== page_stats + +memory page statistics + + +*`linux.memory.page_stats.pgscan_kswapd.pages`*:: ++ +-- +pages scanned by kswapd + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgscan_direct.pages`*:: ++ +-- +pages scanned directly + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgfree.pages`*:: ++ +-- +pages freed by the system + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgsteal_kswapd.pages`*:: ++ +-- +number of pages reclaimed by kswapd + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgsteal_direct.pages`*:: ++ +-- +number of pages reclaimed directly + +type: long + +format: number + +-- + +*`linux.memory.page_stats.direct_efficiency.pct`*:: ++ +-- +direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + +type: scaled_float + +format: percent + +-- + +*`linux.memory.page_stats.kswapd_efficiency.pct`*:: ++ +-- +kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + +type: scaled_float + +format: percent + +-- + +[float] +=== hugepages + +This group contains statistics related to huge pages usage on the system. + + +*`linux.memory.hugepages.total`*:: ++ +-- +Number of huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.used.bytes`*:: ++ +-- +Memory used in allocated huge pages. + + +type: long + +format: bytes + +-- + +*`linux.memory.hugepages.used.pct`*:: ++ +-- +Percentage of huge pages used. + + +type: long + +format: percent + +-- + +*`linux.memory.hugepages.free`*:: ++ +-- +Number of available huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.reserved`*:: ++ +-- +Number of reserved but not allocated huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.surplus`*:: ++ +-- +Number of overcommited huge pages. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.default_size`*:: ++ +-- +Default size for huge pages. + + +type: long + +format: bytes + +-- + [float] === pageinfo @@ -40495,7 +41021,7 @@ type: long *`system.fsstat.total_files`*:: + -- -Total number of files. +Total number of files. Not on Windows. type: long @@ -41357,7 +41883,7 @@ format: bytes *`system.process.memory.rss.bytes`*:: + -- -The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. +The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. Not available on Windows. type: long @@ -41369,7 +41895,31 @@ format: bytes *`system.process.memory.rss.pct`*:: + -- -The percentage of memory the process occupied in main memory (RAM). +The percentage of memory the process occupied in main memory (RAM). Not available on Windows. + + +type: scaled_float + +format: percent + +-- + +*`system.process.memory.wss.bytes`*:: ++ +-- +The Working Set Size. The amount of memory the process occupied in main memory (RAM). Windows only. + + +type: long + +format: bytes + +-- + +*`system.process.memory.wss.pct`*:: ++ +-- +The percentage of memory the process occupied in main memory (RAM). Windows only. type: scaled_float @@ -41381,7 +41931,7 @@ format: percent *`system.process.memory.share`*:: + -- -The shared memory the process uses. +The shared memory the process uses. Not available on Windows. type: long diff --git a/metricbeat/docs/images/metricbeat-azure-app-state-overview.png b/metricbeat/docs/images/metricbeat-azure-app-state-overview.png new file mode 100644 index 00000000000..cffc6e4ef03 Binary files /dev/null and b/metricbeat/docs/images/metricbeat-azure-app-state-overview.png differ diff --git a/metricbeat/docs/images/metricbeat-istio-overview.png b/metricbeat/docs/images/metricbeat-istio-overview.png new file mode 100644 index 00000000000..139fe9260d4 Binary files /dev/null and b/metricbeat/docs/images/metricbeat-istio-overview.png differ diff --git a/metricbeat/docs/modules/aws.asciidoc b/metricbeat/docs/modules/aws.asciidoc index 42d24c65ccd..e02a7a81460 100644 --- a/metricbeat/docs/modules/aws.asciidoc +++ b/metricbeat/docs/modules/aws.asciidoc @@ -19,16 +19,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/metricbeat/docs/modules/azure.asciidoc b/metricbeat/docs/modules/azure.asciidoc index 4db38120041..42d4d619c02 100644 --- a/metricbeat/docs/modules/azure.asciidoc +++ b/metricbeat/docs/modules/azure.asciidoc @@ -45,6 +45,10 @@ The Azure billing dashboards show relevant usage and forecast information: image::./images/metricbeat-azure-billing-overview.png[] +The Azure app_state dashboard shows relevant application insights information: + +image::./images/metricbeat-azure-app-state-overview.png[] + [float] === Module-specific configuration notes @@ -120,6 +124,10 @@ so the `period` for `billing` metricset should be `24h` or multiples of `24h`. === `app_insights` This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. +[float] +=== `app_state` +This metricset concentrate on the most relevant application insights metrics and provides a dashboard for visualization, the `period` (interval) for the `app_state` metricset is set by default at `300s`. + [float] [[azure-api-cost]] == Additional notes about metrics and costs @@ -242,6 +250,14 @@ metricbeat.modules: api_key: '' metrics: - id: ["requests/count", "requests/duration"] + +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' ---- [float] @@ -251,6 +267,8 @@ The following metricsets are available: * <> +* <> + * <> * <> @@ -271,6 +289,8 @@ The following metricsets are available: include::azure/app_insights.asciidoc[] +include::azure/app_state.asciidoc[] + include::azure/billing.asciidoc[] include::azure/compute_vm.asciidoc[] diff --git a/metricbeat/docs/modules/azure/app_state.asciidoc b/metricbeat/docs/modules/azure/app_state.asciidoc new file mode 100644 index 00000000000..10485dc818e --- /dev/null +++ b/metricbeat/docs/modules/azure/app_state.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-azure-app_state]] +[role="xpack"] +=== Azure app_state metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/azure/app_state/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/cloudfoundry.asciidoc b/metricbeat/docs/modules/cloudfoundry.asciidoc index 4d153933c23..614c703d155 100644 --- a/metricbeat/docs/modules/cloudfoundry.asciidoc +++ b/metricbeat/docs/modules/cloudfoundry.asciidoc @@ -117,7 +117,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "". === `shard_id` Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events -from the RLP Gateway. Default: "(generated UUID)". +from the RLP Gateway. [float] ==== `version` @@ -152,6 +152,7 @@ metricbeat.modules: rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 ---- diff --git a/metricbeat/docs/modules/istio.asciidoc b/metricbeat/docs/modules/istio.asciidoc index 52be7886e04..bfda2a9588d 100644 --- a/metricbeat/docs/modules/istio.asciidoc +++ b/metricbeat/docs/modules/istio.asciidoc @@ -8,15 +8,28 @@ This file is generated! See scripts/mage/docs_collector.go beta[] -This is the Istio module. The Istio module collects metrics from the +This is the Istio module. +This module is compatible with versions before `1.5` of Istio where microservices architecture is used. If using +versions priot to `1.5` then `mesh`, `mixer`, `pilot`, `galley`, `citadel` metricsets should be used. +wehre the Istio module collects metrics from the Istio https://istio.io/v1.4/docs/tasks/observability/metrics/querying-metrics/#about-the-prometheus-add-on[prometheus exporters endpoints]. +For versions after `1.5`, `istiod` metricset can be used which collects metrics directly from Istio Daemon. The default metricsets are `mesh`, `mixer`, `pilot`, `galley`, `citadel`. [float] === Compatibility -The Istio module is tested with Istio `1.4`. +The Istio module is tested with Istio `1.4` for `mesh`, `mixer`, `pilot`, `galley`, `citadel`. +The Istio module is tested with Istio `1.7` for `istiod`. + +[float] +=== Dashboard + +The Istio module includes a predefined dashboard with overview information about Istio Daemon. +This dashboard is only compatible with versions of Istio after `1.5` which should be monitored with `istiod` metricset. + +image::./images/metricbeat-istio-overview.png[] [float] @@ -62,6 +75,13 @@ metricbeat.modules: period: 10s # use istio-pilot.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset hosts: ["localhost:15014"] + +# Istio istiod to monitor the Istio Daemon for versions after 1.5 of Istio. +- module: istio + metricsets: ['istiod'] + period: 10s + # use istiod.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset + hosts: ['localhost:15014'] ---- [float] diff --git a/metricbeat/docs/modules/linux.asciidoc b/metricbeat/docs/modules/linux.asciidoc index d63528730cb..ab911885afd 100644 --- a/metricbeat/docs/modules/linux.asciidoc +++ b/metricbeat/docs/modules/linux.asciidoc @@ -22,8 +22,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs @@ -36,13 +38,21 @@ The following metricsets are available: * <> +* <> + * <> +* <> + * <> include::linux/conntrack.asciidoc[] +include::linux/iostat.asciidoc[] + include::linux/ksm.asciidoc[] +include::linux/memory.asciidoc[] + include::linux/pageinfo.asciidoc[] diff --git a/metricbeat/docs/modules/linux/iostat.asciidoc b/metricbeat/docs/modules/linux/iostat.asciidoc new file mode 100644 index 00000000000..7a41297b819 --- /dev/null +++ b/metricbeat/docs/modules/linux/iostat.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-linux-iostat]] +=== linux iostat metricset + +beta[] + +include::../../../module/linux/iostat/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/linux/iostat/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/linux/memory.asciidoc b/metricbeat/docs/modules/linux/memory.asciidoc new file mode 100644 index 00000000000..9ea3d482e57 --- /dev/null +++ b/metricbeat/docs/modules/linux/memory.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-linux-memory]] +=== linux memory metricset + +beta[] + +include::../../../module/linux/memory/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/linux/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/prometheus/query.asciidoc b/metricbeat/docs/modules/prometheus/query.asciidoc index 72c0fde9278..ff746df44a7 100644 --- a/metricbeat/docs/modules/prometheus/query.asciidoc +++ b/metricbeat/docs/modules/prometheus/query.asciidoc @@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-metricset-prometheus-query]] === Prometheus query metricset -beta[] - include::../../../module/prometheus/query/_meta/docs.asciidoc[] diff --git a/metricbeat/docs/modules/prometheus/remote_write.asciidoc b/metricbeat/docs/modules/prometheus/remote_write.asciidoc index 9d871805344..535fdec82e6 100644 --- a/metricbeat/docs/modules/prometheus/remote_write.asciidoc +++ b/metricbeat/docs/modules/prometheus/remote_write.asciidoc @@ -5,8 +5,6 @@ This file is generated! See scripts/mage/docs_collector.go [[metricbeat-metricset-prometheus-remote_write]] === Prometheus remote_write metricset -beta[] - include::../../../module/prometheus/remote_write/_meta/docs.asciidoc[] diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 2232cf3b070..1bc81071e76 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -33,7 +33,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> beta[] |<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | -.10+| .10+| |<> beta[] +.11+| .11+| |<> beta[] +|<> beta[] |<> beta[] |<> |<> @@ -131,7 +132,7 @@ This file is generated! See scripts/mage/docs_collector.go .3+| .3+| |<> beta[] |<> beta[] |<> beta[] -|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +|<> beta[] |image:./images/icon-yes.png[Prebuilt dashboards are available] | .5+| .5+| |<> beta[] |<> beta[] |<> beta[] @@ -175,8 +176,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> beta[] |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.3+| .3+| |<> beta[] +.5+| .5+| |<> beta[] +|<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> @@ -221,8 +224,8 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .3+| .3+| |<> -|<> beta[] -|<> beta[] +|<> +|<> |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .4+| .4+| |<> |<> diff --git a/metricbeat/docs/running-on-cloudfoundry.asciidoc b/metricbeat/docs/running-on-cloudfoundry.asciidoc index 2988e4d3a8b..59802ba505d 100644 --- a/metricbeat/docs/running-on-cloudfoundry.asciidoc +++ b/metricbeat/docs/running-on-cloudfoundry.asciidoc @@ -66,16 +66,8 @@ To check the status, run: $ cf apps name requested state instances memory disk urls -metricbeat started 1/1 256M 1G +metricbeat started 1/1 512M 1G ------------------------------------------------ Metrics should start flowing to Elasticsearch. The events are annotated with metadata added by the <> processor. - - -[WARNING] -======================================= -*Set shard_id to scale:* By default {beatname_uc} will generate a random `shard_id` when it starts. In the case that -{beatname_uc} needs to be scaled passed 1 instance, be sure to set a static `shard_id`. Not setting a static `shard_id` -will result in duplicate events being pushed to Elasticsearch. -======================================= diff --git a/metricbeat/docs/running-on-kubernetes.asciidoc b/metricbeat/docs/running-on-kubernetes.asciidoc index 786977cb294..f852c153e5e 100644 --- a/metricbeat/docs/running-on-kubernetes.asciidoc +++ b/metricbeat/docs/running-on-kubernetes.asciidoc @@ -192,11 +192,6 @@ $ kubectl --namespace=kube-system get ds/metricbeat NAME DESIRED CURRENT READY UP-TO-DATE AVAILABLE NODE-SELECTOR AGE metricbeat 32 32 0 32 0 1m - -$ kubectl --namespace=kube-system get deploy/metricbeat - -NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE -metricbeat 1 1 1 1 1m ------------------------------------------------ Metrics should start flowing to Elasticsearch. diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 39fadeea0e0..b77bb57f9a6 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -95,7 +95,9 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/kvm/status" _ "github.com/elastic/beats/v7/metricbeat/module/linux" _ "github.com/elastic/beats/v7/metricbeat/module/linux/conntrack" + _ "github.com/elastic/beats/v7/metricbeat/module/linux/iostat" _ "github.com/elastic/beats/v7/metricbeat/module/linux/ksm" + _ "github.com/elastic/beats/v7/metricbeat/module/linux/memory" _ "github.com/elastic/beats/v7/metricbeat/module/linux/pageinfo" _ "github.com/elastic/beats/v7/metricbeat/module/logstash" _ "github.com/elastic/beats/v7/metricbeat/module/logstash/node" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9b6f37eb447..9df18e4d7f9 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -572,8 +572,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/module/kafka/broker.go b/metricbeat/module/kafka/broker.go index b9e78c7dacc..2e558d7944a 100644 --- a/metricbeat/module/kafka/broker.go +++ b/metricbeat/module/kafka/broker.go @@ -119,7 +119,7 @@ func (b *Broker) Connect() error { c, err := getClusterWideClient(b.Addr(), b.cfg) if err != nil { closeBroker(b.broker) - return fmt.Errorf("Could not get cluster client for advertised broker with address %v", b.Addr()) + return fmt.Errorf("getting cluster client for advertised broker with address %v: %w", b.Addr(), err) } b.client = c diff --git a/metricbeat/module/kafka/fields.go b/metricbeat/module/kafka/fields.go index 72ee2cdc60c..96c306b5915 100644 --- a/metricbeat/module/kafka/fields.go +++ b/metricbeat/module/kafka/fields.go @@ -32,5 +32,5 @@ func init() { // AssetKafka returns asset data. // This is the base64 encoded gzipped contents of module/kafka. func AssetKafka() string { - return "eJzUWs2O3DYSvs9TFHwaHyyfdg9zWGDXXgQT27HhOECQi8AmS93MSKRMUj3TfvqApKTWL0W1epx4TtOSqr6PxWKxWMVX8ICnO3gg2QO5ATDc5HgHL97Z3y9uABhqqnhpuBR38J8bAAD3DgrJqhxvAPRBKpNSKTK+v4OM5No+VZgj0XgHe6s245gzfefEX4EgBZ4h7Z85lfZTJauyfjKB21fTVbVT8gFV+3hK36xO//c/pwHeSKGrAhX8ZEXhXmRSFcQKwIEcEXaIAhQSBpmSBdzWYgciWM7FvqfSHBBoo89ReZl0PhiOpTseznqPm/HkcgARHFJnWJzdTOIQxhRqPQn2gKdHqYZE4vAIO6IyXCNrIUZzZmTJaWL/H83bGDoA+8XqcTrnMFApqRIq2RhpYNFFGKcKrKpkjFYSZbiVTXrztxbpU6MGOAuiuNGlE1hh+/XAfhP8a4XAGcjMeWx5RhfugbfhMg+/Br8PHSCCuV8eNLlKQKh9t0CjONV+gftQV7/5+cPvHdk2wO3QkMh1XeyQiNCK+mA/AHMgBsyBa8AjCgNcWzRikIGR0Yu1AVX4tUJtEnogQmCefK2wwkTzbxhi8uWAYL9pJqLWAk46LjoNCZRKsopikhGeI0tLVKlGKkUwxlgeihjHwwtCrafRq6FEBZOaPLEsl8QEmWVo6OFyXjTndpqcltZQVlul8Ars+nZbIiWqYocqYK4r2CieQ9A0q5mUOaduN05yJAxVijlS+3uoasTIfw/N927qNsBXguZIRLqWRi13DToatbZUvkn5gFiiShjXVAqB1CzR+EPKd04GaC7tLl0r2+CsYzr4VHK1GGPOVPz3z8PFpmxS5Kd4No3Es9DRJ0HXzJFbQ/XcbuOSy32S5ZU+pBMuN141cg/u60sctE7w0CRcJLuTQd2E1iVYLqgsuNiDlfJR1g7YKbyYhKzMOhayMnt5bRYK/0RqkK2j0khdjUqBWpM96pQH05HeZNQy2+Cv4w4XgF5h+i9AvdZ0r4TeOr0RcA1Uc8Jdl2u35+yJbLt994Pm2y7XiQqvBRe8qAq/ooiBxwOnh37dQKNgup8+aTASyPiIE+MY3g1r7YtpHDmisi5xTuecfMOOQSYVENAlUp5xWp/NNuS7VCq2hV6t4UzwzGWS60qCawNXcz5orObWmT3Hyt4kr13c5CnNSbAQ5JyLPDnnal1pLLOE1CYsKZVFwUdHh9kByyzTaDMWJ2XH22YzKym4IuF2+HedWmO0neODaHsQPIc1H0z9A/flBTF1GEJXVzXnFPWLs81fKJIGos1QuedUKqQ2gt7Bv5N/hew3W0S8Zi0WluuxcxaAUF0WQrXZiKFCr0bbPJnFn67XwkLNdh2PIcZ0JJyO7RE14+Gsvg0DzYfpOaxox+sUjoMU2hJkXNVrFYdzrXfJED6mxVPogLROX8dFLozsFFZ3aLclu5DCDIp+3gVrZp1W2siiGx8NAUYMAW1Ud7FOIjdiE9veSgvkZO8SgXb0r33OQklOK5/xEe0iBeNZhgoFtYHGPNpY069H18Ykgg0NHB7MZDcifijjteu2yJbD6zPDbrMibF9XYFgR3IN8/qs13wtkTd3Cepb1MJdb15n+3JraEIAjY98bT+r+Ldx6w2k0xtLzbBPOXi7H4YPUQ3NdSqSnahawQJviptuHz4VBJUg+2AlrgG4UCoW/1YnIlJL1SUggBl7ip0fCc7LLsdarm1bHnh9RdPpbK31U4CMG3OPyROEXp7gJPMMG2ZBmx2w5ex5CH53iZUIXbKsXzOd5L7Uby/dI7ULbfARhGDV7m6ezLH3f4Bmm8r1vSHAGtz7VH+XGHVPpeQabMtD33HmTBQDO9DyDuhXzDHb47DVPG2LeIkKfBE2XaO2kzMcFqUhm94Jx620aeNYYALgGLmheMWRNg5yLV5ZM265Cu8PB7f2vn6NGotMFH3uWQZi2RbdMcTaBgivM///bnMknKq7yZtOD2LAWuHIS5LfpxGxGl1PmqQ1u3MD1DlRjXrFHrPoax5prT9vrC3y6fFtzueRq1HZOs4fvcVrmu/or6+6faqmpunv77getu5Mmn0t3lT27pa7sGmLxxd0nMiQHUshKuL3Hy9p8WKphu3ixuk4MPaSaf8OUHBcLtHPVdT13FosCLsjTEnBTGY4GDtzsoVKxVKNgUQ2P+Yq9xd7aOUgVGnW6mIhRHFmtqu67bCXkIvI/iJDvXNR9pL95stYuk8YO43trawBXLI8lwKX7blHzfjZuE9Cvcq1Nl1JovJyBl99Agcv0kfBFH2sh719/BCsAhs+kJ/NYq1v7/b6c7/LLyrjKoOmwWsmjbvpEWb08b8RRbfe/AgAA//9LpueC" + return "eJzUWk9v3LYSv/tTDHJyDlFO7x18eMBrUhRumiZIU6DoReCSo13WEqmQ1NqbT1+QlLT6S1Grddr45JU08/txhhwOZ/gKHvB0Bw8keyA3AIabHO/gxTv7+8UNAENNFS8Nl+IO/ncDAODeQSFZleMNgD5IZVIqRcb3d5CRXNunCnMkGu9gb9VmHHOm75z4KxCkwDOk/TOn0n6qZFXWTyZw+2q6qnZKPqBqH0/pm9Xp/35wGuCNFLoqUMFPVhTuRSZVQawAHMgRYYcoQCFhkClZwG0tdiCC5VzseyrNAYE2+hyVl0nng+FYuuPhrPe4GU8uBxDBIXWGxdnNJA5hTKHWk2APeHqUakgkDo+wIyrDNbIWYuQzI0tOE/v/yG9j6ADsZ6vH6ZzDQKWkSqhkY6SBRRdhnCqwqpIxWkmU4VY26flvLdLHRg1wFkRxo0snsML264H9LviXCoEzkJmbseUZXbgH3obLPPwa/DZ0gAjmfnnQ5CoBoZ67BRrFqfYL3Ie6+s3P7//oyLYBboeGRK7rYodEhFbUe/sBmAMxYA5cAx5RGODaohGDDIyMXqwNqMIvFWqT0AMRAvPkS4UVJpp/xRCTzwcE+03jiFoLOOm46DQkUCrJKopJRniOLC1RpRqpFMEYY3koYhwPLwi1nkavhhIVTGryxLJcEhNklqGhh8t50ZxbNzktraGstkrhFdj17bZESlTFDlXAXFewUTyHoGlWMylzTt1unORIGKoUc6T291DViJH/Hprvnes2wFeC5khEupZGLXcNOhq1tlS+SvmAWKJKGNdUCoHULNH4U8p3TgZoLu0uXSvbMFnHdPCp5Goxxpyp+O+fh4tN2aTIT/FsGolnoaNPgq7xkVtDtW+3ccnlPsnySh/SiSk3XjVyD+7rSyZoneChSbhIdieDugmtS7BcUFlwsQcr5aOsHbBTeDEJWZl1LGRl9vLaLBT+hdQgW0elkboalQK1JnvUKQ+mIz1n1DLb4K8zHS4AvYL7L0C9lrtXQm91bwRcA9WccNfl2u05eyLbbt99p/m2y3WiwmvBBS+qwq8oYuDxwOmhXzfQKJjup08ajAQyPuLETAw/DWvti2kcOaKyU+Kczjn5hh2DTCogoEukPOO0PpttyHepVGwLvVrDmeCZyyTXlQTXBq7mfNBYza0ze46VPSevXdzkKc1JsBDkJhd5cpOrnUpjmSWkNmFJqSwKPjo6zA5YZplGm7E4KTveNptZScEVCbfDv+vUGqPtHB9E24PgOaz5YOofuC8viKnDELq6qjmnqF+cbf5CkTQQbYbKPadSIbUR9A7+m/wnZL/ZIuI1a7GwXI+dswCE6rIQqs1GDBV6NdrmySz+dL0WFmq263gMMaYj4XRsj6gZD736Ngw0H6bnsKInXqdwHKTQliDjql6rOJxrvUuG8DEtnkIHpJ30dVzkwshOYXWHdluyCynMoOjnXbDG67TSRhbd+GgIMGIIaKO6i3USuRGb2PZWWiAne5cItKN/7XMWSnJa+YyPaBcpGM8yVCioDTTm0caafj26NiYRbGjg8GAmuxHxQxmvXbdFthxenxl2mxVh+7oCw4rgHuTzf635XiBr6hZ2ZtkZ5nLrOtOfW1MbAnBk7HvjSd2/hVtvOI3GWHqebcLZy+U4fJB6aK5LifRUzQIWaFPcdPvwuTCoBMkHO2EN0I1CofC3OhGZUrI+CQnEwEvm6ZHwnOxyrPXqptWx50cUnf7Wyjkq8BED0+PyROFXp7gJPMMG2ZBmx2w5ex5CH5ziZUIXbKsX+PO8l9qN5VukdqFtPoIwjJq9zdNZlr5v8Ayu/MU3JDiDW5/qj3LjUSPkGVh88pqnacy7TuiToOkSrZ2U+bgcFMnsXjBufa2BZ40BgGvgguYVQ9a0p7l4Zcm0zSK0+wvc3v/2KWokOl3w8LMMwrQNsmWKs+kLXMH/P7YZi08TXN3Lbs6xQSVw4SPIb9N51YyuhsxTG9x3gesdZ8a8Yg849SWKNZeOtp/u+XTxtOZyycWk7Zxmj77jpMj31FdWvT/WUlNV7/bdd1r1Jk02le4qe3JKXdEzeP52t3kMyYEUshIugfCyNhuVatisXaxtE0MPqeZfMSXHxfLoXG1bz52EooAL8rQE3NRlo4ED92qoVCzVKFhUu2G+Xm6xt9btU4VGnS4mYhRHVququx5bCbmI/C8i5PsGdRfnH3bW2mXS2GF8a2wN4IrlsQS4dNssyu9n4zYB/SqXynQphcbLGXj5DRS4TB8JX5xjLeT96w9gBcDwmfRkHmt1Y73fFfM9dlkZV5czHVYredQtlyirl+eNOKrp/XcAAAD//1uFyto=" } diff --git a/metricbeat/module/kafka/partition/_meta/fields.yml b/metricbeat/module/kafka/partition/_meta/fields.yml index 8f241278589..cf40ad266b9 100644 --- a/metricbeat/module/kafka/partition/_meta/fields.yml +++ b/metricbeat/module/kafka/partition/_meta/fields.yml @@ -33,10 +33,6 @@ type: long description: > Leader id (broker). - - name: isr - type: keyword - description: > - List of isr ids. - name: replica type: long description: > diff --git a/metricbeat/module/linux/_meta/config.yml b/metricbeat/module/linux/_meta/config.yml index e00fe571846..490d3245c19 100644 --- a/metricbeat/module/linux/_meta/config.yml +++ b/metricbeat/module/linux/_meta/config.yml @@ -2,8 +2,10 @@ period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/module/linux/_meta/fields.yml b/metricbeat/module/linux/_meta/fields.yml index c3fe9657e89..44236f625e9 100644 --- a/metricbeat/module/linux/_meta/fields.yml +++ b/metricbeat/module/linux/_meta/fields.yml @@ -7,4 +7,5 @@ - name: linux type: group description: > + linux system metrics fields: diff --git a/metricbeat/module/linux/fields.go b/metricbeat/module/linux/fields.go index e4935eb2d7f..90e7e3ed940 100644 --- a/metricbeat/module/linux/fields.go +++ b/metricbeat/module/linux/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLinux returns asset data. // This is the base64 encoded gzipped contents of module/linux. func AssetLinux() string { - return "eJy8l0tv4zYQx+/+FIMcF91Hsm8fCgTdRVEUKYIGe+mhwogcWawpUuWQybqfviBl2YoiOUpTiZcgIfWfn/6ch/IStrRbg1YmfF8BeOU1reEs/X62AnCkCZnWkJPHFYAkFk7VXlmzhh9XANA8C5WVQdMKoFCkJa/T1kswWNFRPi6/q2kNG2dDvf/LgOZ9ma6UsMZ4h2J72BmSjKuP3q7BcM0aEu+DdGE4VBW63b29MZxHQse1lwNbgCmyAwywR6/YK8E/pDMkAYWzzPDT9TcQ1hH3tIagu+DS2T7bkVxbsxnYfAQ+rhrFljwn+ZokyEDg7dFWKFBpFRyNghE6vctmwjtykPFO0RHUW6hwS+CsraCwDgzdgTUPfO2ANgozULZsyoAvqQPtMdfjzhU2GDkDDgchiLkIWu+ACZ0oSY6+fkujNsYOXPP/l2JMZAC1I5S75BEJ31wk9u65X54dSMPkfBaTkuaw7rdQ5eRiObd3eleSI9CK/T54+6NhgBOot6jVMyEfyj9w1JfoQaAx1kNOkFwc8ObQAFM+ZI7Yo/MzWJhyHrS121BH+5QoocR0zznBPu6x0zTHHbH6p5OcLe2Wqzmmxn3Zk/PCo+/XzH+eFr/eXHUmwxMHQI0b4oxLdLNk/k0SbqLERhaYXk1gUYPxng2jPDHs9Ztoj8EEM5s134z6O9AJjAoPGLdWo1cDTf/5GNfpakSJZhNd8dZCgez3hdW8/bhLcRpkLNDMMQCPbdOrihgqcptU1006lXhLkMdmFQHMKUxO/SAzVlImSlSz4DZOpsaZ0BxhGpEVfs8icZvZ0zBlqOf1VIZaK4GxacYO0svDbhkoU9g5+uWA9qmmmQcpd1nvgXGgCYZ8QY9QOFvB69pZ8TpFiAEasVgPgQ/tK9+BdZLcEzvsl6vL0XscYp7AndivLhNXeomBE2NYXbSzN2eD+4+k2UTCROGIQJTBbDnm3MWfb15cX/78Nbv55Y+vp9HOF0c7n4p2sTjaxVS0t4ujvZ2K9m5xtHdT0d4vjvZ+KtqHxdE+TEX7uDjax6lonxZH+zQV7fPiaJ8nt9zlx8H52DxooeK3EL96MTjxbf4Xif4/mxNIfsc7QK2twHgK0sDvfAXEqRoD3PvSWP0bAAD//4Pi/m8=" + return "eJzEmd9v2zgSx9/zVwwCHNAWVzfp7/rhgNylOBR3uQu27csudoUxObK5pkiVP+y6f/2CpBXLtmQrcaT6pWgkz3z4neHMkH4Oc1qNQQrlv58BOOEkjeE8/v/8DMCQJLQ0hgk5PAPgZJkRpRNajeEfZwCQvguF5l7SGUAuSHI7jo+eg8KCNubDx61KGsPUaF+u/9Jgc2PXrqyjAgpyRjC7flj3UffDtFLOIJvfPWnyFz6766o+LSzh02R8F6QOY31RoFltPWvDOeI6fNbmQOeg8uwOBqxDJ6wTzP49vkMckBltLfzr9iswbcju2GqCroNzo3fZNuRSq2nDwyPw4VMim5Oz0XxJHLgncHojK+QopPCGWsEIjVxlPeFtOEg5I2gD6jQUOCcwWheQawOKlqDVnq410GShB8qKTShwM6pBO5zIduVy7RXvAcd6xsja3Eu5Akto2Ix46/IrGjFVuiHMj5dilkgBSkPIV1EjYi4FEnfivLs9a5DKknFZSErqQ7r/+WJCJmznKqbLGRkCKaxbO6/+SQxwAHWBUpwIuW9+T1E3QwcMldIOJgRRxQZt7gpgzIfMkHVoXA8SxpwHqfXcl0E+wWYwwxjnCcHa76bSpNcNWfGjlpx3IupQSftoHHuWD3WNkLMjQ988WTcqyEzJZiWZzBJr7CS51Lir7VHpvswI1F3+BZewdmkh+uRQkgFLTCuewr4MufnNk0/7KBQfTgvBaNS4jKURjgZeR/T52AvZiseggdjQCmv3aGvr6hCAYZU/jTwqvgYeTVZur5E8GvY/g/Gkem500cw42p2etCnQjWGfbGsBuESxi3ciOC7I4JTAiYLAlqRcnEa2s6ZR8VQQLZkF8UP7dUDVU8o8puxpCcPpvpP0DxS+2qG4mGahMfWDHizDE6GSfE9DGAJlxx3bTB5raM/c0QdIUlM3exToIbflaYkRnglGWTDbU1IkDwk8JEchpBRp+9mncRGfXvz/NL0n3jafgB9Ef0uGkXIBXufxfBvZuTdCTdcD4BZyexN6MkHFl4K7GXgnpPiBwW1c9OatpyO4Tq9bdN6kVzRj3sRpPUzEwsICpQ9egEltY2gvLy7+ttFjb9Sc26KPOXPb7MGrCYeuub4/4GLiP59vapcQ97xrKDEMhnaGppdD1udoOHkJZ2ZvabexNLGIRn8nw4jQ+Nb2k7djMF71Js1XJb55OoBR4B3GQkt0ouF+4XSM2xgaNkM1Dao4rSFH66oCGVffrlLupcwsQ9XHXcvmhB6qTDpQxCNkSqcZLggm4VwcANQhTBuPnpnSnDI2Q9ELblIyFumIZgjjbUyB37NAXGV2N0zuy3415b6UgmE4n4cKspOHFVJBhd66pHm0avnfdG0d7QPHra8dqp1JyocX0LXHMnbgB5fNaci5bG6XWPJR1O6+waoG6jSYH4tmSvl1osNkBcn1MUAuDDE3PGDyK9tv98ppboiGAwveom5huEi/aRzSzhHKAaO7OZwlWkNMoii6RjrSDhfqdtqjYU8vZJTngglSbDUqWfvNpGUoiWdNo2qdukxT6THs5LuihQ1DZQCnNIIrkHpJpvY3EIrHQmlryRPGTeuMn05lapt3dlN9aS/yKZw/R4Lk+6dIUC1/5qfUlKPt1bs0lIvvYzj/Larw+/mh6v4lHAiiFWBaudDqa1U+9Clc/xIRQNYJ7G080aja4vauP440BKcdyn633bGGXlvQ+tepUmvZnojeEm+8aerM3fblDtg3qQsHhkCLUuo0i2xWcYT80K45wt1xt7SMeVtn4K0sOjgpG7r3+P64+YELFDLOz/fNFEPpouTn8lcUMPEOlHaNSdNtQdabUvqe2+Sx9egFGaaLQnRNe045eumarvs6o5+wZa+T+3SXmWvTyFyf0oXKdR8nhwbbhw4ME8/5Ktv5QjtQBzWu0WG6On9RGs1eRA/BQTIWumEsbCkjJyvQhu/ly7GOcn1z1RrkJuYO3JH95iqdfa63j1zHsOpo5xe7PXgbryUHOxJGCkMEbObV3Iat8vKPi2e3V//+mH3+9OvHw2iXg6NddkV7OTjay65orwZHe9UV7fXgaK+7or0ZHO1NV7S3g6O97Yr2bnC0d13R3g+O9r4r2ofB0T50LrnDt4PLtn5QQSnNyY6eNXZ8PfmT9g4PHUh+wWU1cwqtIDb82hQQumpwsDVpnP0VAAD//2DzKvg=" } diff --git a/metricbeat/module/linux/iostat/_meta/data.json b/metricbeat/module/linux/iostat/_meta/data.json new file mode 100644 index 00000000000..99f23d8cc95 --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/data.json @@ -0,0 +1,49 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "linux.iostat", + "duration": 115000, + "module": "linux" + }, + "linux": { + "iostat": { + "await": 0, + "busy": 0, + "name": "sr0", + "queue": { + "avg_size": 0 + }, + "read": { + "await": 0, + "per_sec": { + "bytes": 0 + }, + "request": { + "merges_per_sec": 0, + "per_sec": 0 + } + }, + "request": { + "avg_size": 0 + }, + "service_time": 0, + "write": { + "await": 0, + "per_sec": { + "bytes": 0 + }, + "request": { + "merges_per_sec": 0, + "per_sec": 0 + } + } + } + }, + "metricset": { + "name": "iostat", + "period": 10000 + }, + "service": { + "type": "linux" + } +} \ No newline at end of file diff --git a/metricbeat/module/linux/iostat/_meta/docs.asciidoc b/metricbeat/module/linux/iostat/_meta/docs.asciidoc new file mode 100644 index 00000000000..ec4c4de1618 --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +The iostat module reports per-disk IO statistics that emulate `iostat -x` on linux. + +NOTE: as of now, this data is part of system/diskio on Metricbeat, but can only be found in the Linux integration in Fleet. In the future, this data will be removed from system/memory. \ No newline at end of file diff --git a/metricbeat/module/linux/iostat/_meta/fields.yml b/metricbeat/module/linux/iostat/_meta/fields.yml new file mode 100644 index 00000000000..a7ded63f8fc --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/fields.yml @@ -0,0 +1,61 @@ +- name: iostat + type: group + release: beta + description: > + iostat + fields: + - name: read.request.merges_per_sec + type: float + description: > + The number of read requests merged per second that were queued to the device. + - name: write.request.merges_per_sec + type: float + description: > + The number of write requests merged per second that were queued to the device. + - name: read.request.per_sec + type: float + description: > + The number of read requests that were issued to the device per second + - name: write.request.per_sec + type: float + description: > + The number of write requests that were issued to the device per second + - name: read.per_sec.bytes + type: float + description: > + The number of Bytes read from the device per second. + format: bytes + - name: read.await + type: float + description: > + The average time spent for read requests issued to the device to be served. + - name: write.per_sec.bytes + type: float + description: > + The number of Bytes write from the device per second. + format: bytes + - name: write.await + type: float + description: > + The average time spent for write requests issued to the device to be served. + - name: request.avg_size + type: float + description: > + The average size (in bytes) of the requests that were issued to the device. + - name: queue.avg_size + type: float + description: > + The average queue length of the requests that were issued to the device. + - name: await + type: float + description: > + The average time spent for requests issued to the device to be served. + - name: service_time + type: float + description: > + The average service time (in milliseconds) for I/O requests that were issued to the device. + - name: busy + type: float + description: > + Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + diff --git a/metricbeat/module/linux/iostat/data.go b/metricbeat/module/linux/iostat/data.go new file mode 100644 index 00000000000..4e546deaa34 --- /dev/null +++ b/metricbeat/module/linux/iostat/data.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iostat + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" +) + +// AddLinuxIOStat adds the linux iostat data to the provided map +func AddLinuxIOStat(extraMetrics diskio.IOMetric) common.MapStr { + return common.MapStr{ + "read": common.MapStr{ + "request": common.MapStr{ + "merges_per_sec": extraMetrics.ReadRequestMergeCountPerSec, + "per_sec": extraMetrics.ReadRequestCountPerSec, + }, + "per_sec": common.MapStr{ + "bytes": extraMetrics.ReadBytesPerSec, + }, + "await": extraMetrics.AvgReadAwaitTime, + }, + "write": common.MapStr{ + "request": common.MapStr{ + "merges_per_sec": extraMetrics.WriteRequestMergeCountPerSec, + "per_sec": extraMetrics.WriteRequestCountPerSec, + }, + "per_sec": common.MapStr{ + "bytes": extraMetrics.WriteBytesPerSec, + }, + "await": extraMetrics.AvgWriteAwaitTime, + }, + "queue": common.MapStr{ + "avg_size": extraMetrics.AvgQueueSize, + }, + "request": common.MapStr{ + "avg_size": extraMetrics.AvgRequestSize, + }, + "await": extraMetrics.AvgAwaitTime, + "service_time": extraMetrics.AvgServiceTime, + "busy": extraMetrics.BusyPct, + } +} diff --git a/metricbeat/module/linux/iostat/iostat.go b/metricbeat/module/linux/iostat/iostat.go new file mode 100644 index 00000000000..049b4daadf7 --- /dev/null +++ b/metricbeat/module/linux/iostat/iostat.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iostat + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("linux", "iostat", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + stats *diskio.IOStat + includeDevices []string +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The linux iostat metricset is beta.") + + config := struct { + IncludeDevices []string `config:"iostat.include_devices"` + }{IncludeDevices: []string{}} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + includeDevices: config.IncludeDevices, + stats: diskio.NewDiskIOStat(), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + + IOstats, err := diskio.IOCounters(m.includeDevices...) + if err != nil { + return errors.Wrap(err, "disk io counters") + } + + // Sample the current cpu counter + m.stats.OpenSampling() + + // Store the last cpu counter when finished + defer m.stats.CloseSampling() + + for _, counters := range IOstats { + event := common.MapStr{ + "name": counters.Name, + } + if counters.SerialNumber != "" { + event["serial_number"] = counters.SerialNumber + } + result, err := m.stats.CalcIOStatistics(counters) + if err != nil { + return errors.Wrap(err, "error calculating iostat") + } + IOstats := AddLinuxIOStat(result) + event.DeepUpdate(IOstats) + + isOpen := report.Event(mb.Event{ + MetricSetFields: event, + }) + if !isOpen { + return nil + } + } + return nil +} diff --git a/metricbeat/module/linux/iostat/iostat_test.go b/metricbeat/module/linux/iostat/iostat_test.go new file mode 100644 index 00000000000..6a607fc66d4 --- /dev/null +++ b/metricbeat/module/linux/iostat/iostat_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package iostat + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("linux", "iostat").Fields.StringToPrint()) +} + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + err := mbtest.WriteEventsReporterV2Error(f, t, ".") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "linux", + "metricsets": []string{"iostat"}, + } +} diff --git a/metricbeat/module/linux/memory/_meta/data.json b/metricbeat/module/linux/memory/_meta/data.json new file mode 100644 index 00000000000..79a3d431869 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/data.json @@ -0,0 +1,59 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "linux.memory", + "duration": 115000, + "module": "linux" + }, + "linux": { + "memory": { + "hugepages": { + "default_size": 2097152, + "free": 0, + "reserved": 0, + "surplus": 0, + "swap": { + "out": { + "fallback": 0, + "pages": 0 + } + }, + "total": 0, + "used": { + "bytes": 0, + "pct": 0 + } + }, + "page_stats": { + "direct_efficiency": { + "pct": 0.9228 + }, + "kswapd_efficiency": { + "pct": 0.7523 + }, + "pgfree": { + "pages": 16061818710 + }, + "pgscan_direct": { + "pages": 1198580 + }, + "pgscan_kswapd": { + "pages": 50222460 + }, + "pgsteal_direct": { + "pages": 1106083 + }, + "pgsteal_kswapd": { + "pages": 37782783 + } + } + } + }, + "metricset": { + "name": "memory", + "period": 10000 + }, + "service": { + "type": "linux" + } +} \ No newline at end of file diff --git a/metricbeat/module/linux/memory/_meta/docs.asciidoc b/metricbeat/module/linux/memory/_meta/docs.asciidoc new file mode 100644 index 00000000000..8b0bbaed750 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +The memory metricset extends system/memory and adds linux-specific memory metrics, including Huge Pages and overall paging statistics. + +NOTE: as of now, this data is part of system/memory on Metricbeat, but can only be found in the Linux integration in Fleet. In the future, this data will be removed from system/memory. \ No newline at end of file diff --git a/metricbeat/module/linux/memory/_meta/fields.yml b/metricbeat/module/linux/memory/_meta/fields.yml new file mode 100644 index 00000000000..2490e69aa21 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/fields.yml @@ -0,0 +1,78 @@ +- name: memory + type: group + release: beta + description: > + Linux memory data + fields: + - name: page_stats + type: group + description: memory page statistics + fields: + - name: pgscan_kswapd.pages + type: long + format: number + description: pages scanned by kswapd + - name: pgscan_direct.pages + type: long + format: number + description: pages scanned directly + - name: pgfree.pages + type: long + format: number + description: pages freed by the system + - name: pgsteal_kswapd.pages + type: long + format: number + description: number of pages reclaimed by kswapd + - name: pgsteal_direct.pages + type: long + format: number + description: number of pages reclaimed directly + - name: direct_efficiency.pct + type: scaled_float + format: percent + description: direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + - name: kswapd_efficiency.pct + type: scaled_float + format: percent + description: kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + - name: hugepages + type: group + prefix: "[float]" + description: This group contains statistics related to huge pages usage on the system. + fields: + - name: total + type: long + format: number + description: > + Number of huge pages in the pool. + - name: used.bytes + type: long + format: bytes + description: > + Memory used in allocated huge pages. + - name: used.pct + type: long + format: percent + description: > + Percentage of huge pages used. + - name: free + type: long + format: number + description: > + Number of available huge pages in the pool. + - name: reserved + type: long + format: number + description: > + Number of reserved but not allocated huge pages in the pool. + - name: surplus + type: long + format: number + description: > + Number of overcommited huge pages. + - name: default_size + type: long + format: bytes + description: > + Default size for huge pages. diff --git a/metricbeat/module/linux/memory/data.go b/metricbeat/module/linux/memory/data.go new file mode 100644 index 00000000000..d3f1a5ef2f8 --- /dev/null +++ b/metricbeat/module/linux/memory/data.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package memory + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + mem "github.com/elastic/beats/v7/libbeat/metric/system/memory" +) + +// FetchLinuxMemStats gets page_stat and huge pages data for linux +func FetchLinuxMemStats(baseMap common.MapStr) error { + + vmstat, err := mem.GetVMStat() + if err != nil { + return errors.Wrap(err, "VMStat") + } + + if vmstat != nil { + pageStats := common.MapStr{ + "pgscan_kswapd": common.MapStr{ + "pages": vmstat.PgscanKswapd, + }, + "pgscan_direct": common.MapStr{ + "pages": vmstat.PgscanDirect, + }, + "pgfree": common.MapStr{ + "pages": vmstat.Pgfree, + }, + "pgsteal_kswapd": common.MapStr{ + "pages": vmstat.PgstealKswapd, + }, + "pgsteal_direct": common.MapStr{ + "pages": vmstat.PgstealDirect, + }, + } + // This is similar to the vmeff stat gathered by sar + // these ratios calculate thhe efficiency of page reclaim + if vmstat.PgscanDirect != 0 { + pageStats["direct_efficiency"] = common.MapStr{ + "pct": common.Round(float64(vmstat.PgstealDirect)/float64(vmstat.PgscanDirect), common.DefaultDecimalPlacesCount), + } + } + + if vmstat.PgscanKswapd != 0 { + pageStats["kswapd_efficiency"] = common.MapStr{ + "pct": common.Round(float64(vmstat.PgstealKswapd)/float64(vmstat.PgscanKswapd), common.DefaultDecimalPlacesCount), + } + } + baseMap["page_stats"] = pageStats + } + + hugePagesStat, err := mem.GetHugeTLBPages() + if err != nil { + return errors.Wrap(err, "hugepages") + } + if hugePagesStat != nil { + mem.AddHugeTLBPagesPercentage(hugePagesStat) + thp := common.MapStr{ + "total": hugePagesStat.Total, + "used": common.MapStr{ + "bytes": hugePagesStat.TotalAllocatedSize, + "pct": hugePagesStat.UsedPercent, + }, + "free": hugePagesStat.Free, + "reserved": hugePagesStat.Reserved, + "surplus": hugePagesStat.Surplus, + "default_size": hugePagesStat.DefaultSize, + } + if vmstat != nil { + thp["swap"] = common.MapStr{ + "out": common.MapStr{ + "pages": vmstat.ThpSwpout, + "fallback": vmstat.ThpSwpoutFallback, + }, + } + } + baseMap["hugepages"] = thp + } + return nil +} diff --git a/metricbeat/module/linux/memory/memory.go b/metricbeat/module/linux/memory/memory.go new file mode 100644 index 00000000000..163e3bea771 --- /dev/null +++ b/metricbeat/module/linux/memory/memory.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memory + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("linux", "memory", New, mb.DefaultMetricSet()) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The linux memory metricset is beta.") + + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + rootEvent := common.MapStr{} + FetchLinuxMemStats(rootEvent) + report.Event(mb.Event{ + MetricSetFields: rootEvent, + }) + return nil +} diff --git a/metricbeat/module/linux/memory/memory_test.go b/metricbeat/module/linux/memory/memory_test.go new file mode 100644 index 00000000000..2980c94841e --- /dev/null +++ b/metricbeat/module/linux/memory/memory_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package memory + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("linux", "memory").Fields.StringToPrint()) +} + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + err := mbtest.WriteEventsReporterV2Error(f, t, ".") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "linux", + "metricsets": []string{"memory"}, + } +} diff --git a/metricbeat/module/prometheus/fields.go b/metricbeat/module/prometheus/fields.go index e93b578c7b7..ff89ddf8ac7 100644 --- a/metricbeat/module/prometheus/fields.go +++ b/metricbeat/module/prometheus/fields.go @@ -32,5 +32,5 @@ func init() { // AssetPrometheus returns asset data. // This is the base64 encoded gzipped contents of module/prometheus. func AssetPrometheus() string { - return "eJzMkkGO2zAMRfc+xYe7G2TmAF70BAXaosuiCBT7O1ZHllSSniC3LxzHGU0yQJF2Uy75RfLxi4945rFBljTSBk5aAeYtsEH95ZKsK6CjtuKz+RQbfKwA4Js5U2grLrNDL2mEw2sVGLucfLSnCtAhiW3bFHu/b9C7oKwAYaBTNti7+Q3NfNxrg++1aqg3qAezXP+ogN4zdNqc5j4iupFX1HPYMc+9JE35nCnL5viAz9JR4BV+zEnMRcNA4QbB7RgUBx8CRmftgN6L2gY2EEI1OCG6NO0CL/1WlKX46eEirDBp95OtFeklsV3UZx4PSbpCfsfmNQpnR5r49jz1BmZR76e52u2Nuh1dzj7uz0/rh/ovoW9of02U4//G+uLCdPr1Kdh627P89VPB//Z639nqZqfyNP8Ac2qwfiVLHy5jdzRX5K9vfUURjsm4PYg3/gvR0genPivYqzNn45TyQrmD9ncAAAD//1baTA8=" + return "eJzMkk1u20AMhfc6xYO6C5wcQIueoEBbdFkUxlh6sqaZv5JUDN++kGU5im2gf5tyyTckP77hI555bFAkR9rAUSvAvAU2qD9dknUFdNRWfDGfU4P3FQB8MWcKbcUVduglRzi8VoGpK9kne6oAHbLYts2p9/sGvQvKChAGOmWDvZve0MynvTb4WquGeoN6MCv1twroPUOnzWnuI5KLvKKewo5l6iV5LOfMumyKd/goHQVe4WPJYi4ZBgo3CG7HoDj4EBCdtQN6L2ob2EAI1eCE6PK4C7z0W1Dm4qeHi7DA5N13trZKz4ntrD7zeMjSreQ7Ni+xcjbSxLfnqTcws/rnNFe7vVG30ZXi0/78tH6o/xL6hvbHSDn+b6wvLoynXx+DLbc9yZ8/rPjfXu+drW52Wp/mL2BODZav5NqHe2NvL30BEcZs3B7EG/+FZ+6DU58F69WXs21KeaH8NuvPAAAA///rUkpn" } diff --git a/metricbeat/module/prometheus/query/_meta/fields.yml b/metricbeat/module/prometheus/query/_meta/fields.yml index acbc35db449..31f29117385 100644 --- a/metricbeat/module/prometheus/query/_meta/fields.yml +++ b/metricbeat/module/prometheus/query/_meta/fields.yml @@ -2,5 +2,5 @@ type: group description: > query metricset - release: beta + release: ga fields: diff --git a/metricbeat/module/prometheus/remote_write/_meta/fields.yml b/metricbeat/module/prometheus/remote_write/_meta/fields.yml index e3b2f050a40..17f5f5fe801 100644 --- a/metricbeat/module/prometheus/remote_write/_meta/fields.yml +++ b/metricbeat/module/prometheus/remote_write/_meta/fields.yml @@ -2,5 +2,5 @@ type: group description: > remote write metrics from Prometheus server - release: beta + release: ga fields: diff --git a/metricbeat/module/system/cpu/_meta/data.json b/metricbeat/module/system/cpu/_meta/data.json index 4a5fd7c8ff7..cb9548ec6e4 100644 --- a/metricbeat/module/system/cpu/_meta/data.json +++ b/metricbeat/module/system/cpu/_meta/data.json @@ -7,7 +7,7 @@ }, "host": { "cpu": { - "pct": 0.0816 + "pct": 0.1629 } }, "metricset": { @@ -19,68 +19,68 @@ }, "system": { "cpu": { - "cores": 12, + "cores": 4, "idle": { "norm": { - "pct": 0.9184 + "pct": 0.8371 }, - "pct": 11.0208, - "ticks": 1964402 + "pct": 3.3484, + "ticks": 806327077 }, "iowait": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 5083 + "ticks": 312229 }, "irq": { "norm": { - "pct": 0 + "pct": 0.0125 }, - "pct": 0, - "ticks": 0 + "pct": 0.0501, + "ticks": 6548016 }, "nice": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 9752 + "ticks": 517231 }, "softirq": { "norm": { - "pct": 0.0058 + "pct": 0.0025 }, - "pct": 0.0699, - "ticks": 10386 + "pct": 0.01, + "ticks": 1561223 }, "steal": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 0 + "ticks": 156646 }, "system": { "norm": { - "pct": 0.005 + "pct": 0.0551 }, - "pct": 0.06, - "ticks": 22274 + "pct": 0.2206, + "ticks": 31037256 }, "total": { "norm": { - "pct": 0.0816 + "pct": 0.1629 }, - "pct": 0.9792 + "pct": 0.6516 }, "user": { "norm": { - "pct": 0.0708 + "pct": 0.0927 }, - "pct": 0.8493, - "ticks": 123767 + "pct": 0.3709, + "ticks": 139619320 } } } diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 7333df6dec7..7650b65cd80 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -20,11 +20,8 @@ package cpu import ( - "strings" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/metric/system/cpu" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -69,50 +66,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to fetch CPU times") } - event := common.MapStr{"cores": cpu.NumCores} - hostFields := common.MapStr{} - for _, metric := range m.config.Metrics { - switch strings.ToLower(metric) { - case percentages: - pct := sample.Percentages() - event.Put("user.pct", pct.User) - event.Put("system.pct", pct.System) - event.Put("idle.pct", pct.Idle) - event.Put("iowait.pct", pct.IOWait) - event.Put("irq.pct", pct.IRQ) - event.Put("nice.pct", pct.Nice) - event.Put("softirq.pct", pct.SoftIRQ) - event.Put("steal.pct", pct.Steal) - event.Put("total.pct", pct.Total) - case normalizedPercentages: - normalizedPct := sample.NormalizedPercentages() - event.Put("user.norm.pct", normalizedPct.User) - event.Put("system.norm.pct", normalizedPct.System) - event.Put("idle.norm.pct", normalizedPct.Idle) - event.Put("iowait.norm.pct", normalizedPct.IOWait) - event.Put("irq.norm.pct", normalizedPct.IRQ) - event.Put("nice.norm.pct", normalizedPct.Nice) - event.Put("softirq.norm.pct", normalizedPct.SoftIRQ) - event.Put("steal.norm.pct", normalizedPct.Steal) - event.Put("total.norm.pct", normalizedPct.Total) - hostFields.Put("host.cpu.pct", normalizedPct.Total) - case ticks: - ticks := sample.Ticks() - event.Put("user.ticks", ticks.User) - event.Put("system.ticks", ticks.System) - event.Put("idle.ticks", ticks.Idle) - event.Put("iowait.ticks", ticks.IOWait) - event.Put("irq.ticks", ticks.IRQ) - event.Put("nice.ticks", ticks.Nice) - event.Put("softirq.ticks", ticks.SoftIRQ) - event.Put("steal.ticks", ticks.Steal) - } - } - - r.Event(mb.Event{ - RootFields: hostFields, - MetricSetFields: event, - }) + r.Event(collectCPUMetrics(m.config.Metrics, sample)) return nil } diff --git a/metricbeat/module/system/cpu/data.go b/metricbeat/module/system/cpu/data.go new file mode 100644 index 00000000000..497a91a6173 --- /dev/null +++ b/metricbeat/module/system/cpu/data.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package cpu + +import ( + "runtime" + "strings" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/cpu" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// CPU metrics are highly OS-specific, so we need to build the event per-OS +func getPlatformCPUMetrics(sample *cpu.Metrics, selectors []string, event common.MapStr) { + for _, metric := range selectors { + switch strings.ToLower(metric) { + case percentages: + pct := sample.Percentages() + event.Put("user.pct", pct.User) + event.Put("system.pct", pct.System) + event.Put("idle.pct", pct.Idle) + event.Put("total.pct", pct.Total) + + if runtime.GOOS != "windows" { + event.Put("nice.pct", pct.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.pct", pct.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.pct", pct.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.pct", pct.SoftIRQ) + event.Put("steal.pct", pct.Steal) + } + case normalizedPercentages: + normalizedPct := sample.NormalizedPercentages() + event.Put("user.norm.pct", normalizedPct.User) + event.Put("system.norm.pct", normalizedPct.System) + event.Put("idle.norm.pct", normalizedPct.Idle) + event.Put("total.norm.pct", normalizedPct.Total) + + if runtime.GOOS != "windows" { + event.Put("nice.norm.pct", normalizedPct.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.norm.pct", normalizedPct.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.norm.pct", normalizedPct.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.norm.pct", normalizedPct.SoftIRQ) + event.Put("steal.norm.pct", normalizedPct.Steal) + } + case ticks: + ticks := sample.Ticks() + event.Put("user.ticks", ticks.User) + event.Put("system.ticks", ticks.System) + event.Put("idle.ticks", ticks.Idle) + + if runtime.GOOS != "windows" { + event.Put("nice.ticks", ticks.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.ticks", ticks.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.ticks", ticks.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.ticks", ticks.SoftIRQ) + event.Put("steal.ticks", ticks.Steal) + } + } + } +} + +// gather CPU metrics +func collectCPUMetrics(selectors []string, sample *cpu.Metrics) mb.Event { + event := common.MapStr{"cores": cpu.NumCores} + getPlatformCPUMetrics(sample, selectors, event) + + //generate the host fields here, since we don't want users disabling it. + normalizedPct := sample.NormalizedPercentages() + hostFields := common.MapStr{} + hostFields.Put("host.cpu.pct", normalizedPct.Total) + + return mb.Event{ + RootFields: hostFields, + MetricSetFields: event, + } +} diff --git a/metricbeat/module/system/diskio/_meta/data.json b/metricbeat/module/system/diskio/_meta/data.json index b9c8533b0c8..a2301ef8888 100644 --- a/metricbeat/module/system/diskio/_meta/data.json +++ b/metricbeat/module/system/diskio/_meta/data.json @@ -15,7 +15,7 @@ "system": { "diskio": { "io": { - "time": 364 + "time": 601740 }, "iostat": { "await": 0, @@ -48,16 +48,16 @@ } } }, - "name": "loop1", + "name": "sdb1", "read": { - "bytes": 5267456, - "count": 4124, - "time": 557 + "bytes": 25128030208, + "count": 3146154, + "time": 833872 }, "write": { - "bytes": 0, - "count": 0, - "time": 0 + "bytes": 34401640448, + "count": 861040, + "time": 11224168 } } } diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 9da3a3c2344..1359180cff6 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -20,9 +20,14 @@ package diskio import ( + "fmt" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/linux/iostat" + "github.com/elastic/beats/v7/metricbeat/module/system" "github.com/pkg/errors" ) @@ -36,9 +41,10 @@ func init() { // MetricSet for fetching system disk IO metrics. type MetricSet struct { mb.BaseMetricSet - statistics *DiskIOStat + statistics *diskio.IOStat includeDevices []string prevCounters diskCounter + IsAgent bool } // diskCounter stores previous disk counter values for calculating gauges in next collection @@ -57,17 +63,23 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + return &MetricSet{ BaseMetricSet: base, - statistics: NewDiskIOStat(), + statistics: diskio.NewDiskIOStat(), includeDevices: config.IncludeDevices, prevCounters: diskCounter{}, + IsAgent: systemModule.IsAgent, }, nil } // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch(r mb.ReporterV2) error { - stats, err := IOCounters(m.includeDevices...) + stats, err := diskio.IOCounters(m.includeDevices...) if err != nil { return errors.Wrap(err, "disk io counters") } @@ -101,40 +113,13 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { diskReadBytes += counters.ReadBytes diskWriteBytes += counters.WriteBytes - var extraMetrics DiskIOMetric - err := m.statistics.CalIOStatistics(&extraMetrics, counters) - if err == nil { - event["iostat"] = common.MapStr{ - "read": common.MapStr{ - "request": common.MapStr{ - "merges_per_sec": extraMetrics.ReadRequestMergeCountPerSec, - "per_sec": extraMetrics.ReadRequestCountPerSec, - }, - "per_sec": common.MapStr{ - "bytes": extraMetrics.ReadBytesPerSec, - }, - "await": extraMetrics.AvgReadAwaitTime, - }, - "write": common.MapStr{ - "request": common.MapStr{ - "merges_per_sec": extraMetrics.WriteRequestMergeCountPerSec, - "per_sec": extraMetrics.WriteRequestCountPerSec, - }, - "per_sec": common.MapStr{ - "bytes": extraMetrics.WriteBytesPerSec, - }, - "await": extraMetrics.AvgWriteAwaitTime, - }, - "queue": common.MapStr{ - "avg_size": extraMetrics.AvgQueueSize, - }, - "request": common.MapStr{ - "avg_size": extraMetrics.AvgRequestSize, - }, - "await": extraMetrics.AvgAwaitTime, - "service_time": extraMetrics.AvgServiceTime, - "busy": extraMetrics.BusyPct, + //Add linux-only data if agent is off as not to make breaking changes. + if !m.IsAgent { + result, err := m.statistics.CalcIOStatistics(counters) + if err != nil { + return errors.Wrap(err, "error calculating iostat") } + event["iostat"] = iostat.AddLinuxIOStat(result) } if counters.SerialNumber != "" { diff --git a/metricbeat/module/system/diskio/diskio_test.go b/metricbeat/module/system/diskio/diskio_test.go index aabc44c374b..b5494ac4bfe 100644 --- a/metricbeat/module/system/diskio/diskio_test.go +++ b/metricbeat/module/system/diskio/diskio_test.go @@ -27,8 +27,48 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/system" ) +func TestDataNameFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + "diskio.include_devices": []string{"sda", "sda1", "sda2"}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, conf) + data, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, errs) + assert.Equal(t, 3, len(data)) +} + +func TestDataEmptyFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, conf) + data, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, errs) + assert.Equal(t, 10, len(data)) +} + func TestFetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) events, errs := mbtest.ReportingFetchV2Error(f) diff --git a/metricbeat/module/system/fields.go b/metricbeat/module/system/fields.go index 28135f5fa95..d78002d2847 100644 --- a/metricbeat/module/system/fields.go +++ b/metricbeat/module/system/fields.go @@ -32,5 +32,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded gzipped contents of module/system. func AssetSystem() string { - return "eJzsff+PGzey5+/5KwgfFhm/m5E93mxe3vxwgON5uRvAWQ9sB+8Bh4NMdZck7rDJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/VSwW6xuLN+QJdndE77SB5DtCDDMc7siLT/iLF98REoOOFEsNk+KO/K/vCCHE/ZFoQ02mSQJGsUhfE86egLx7/I1QEZMEEql2JNN0BdfErKkhVAGJJOcQGYjJUsmEmDUQmYKihomVRzH7jhC9lsrMIymWbHVHjMrgO0IUcKAa7siKfkfIkgGP9R0CuiGCJlAhw/6YXWo/q2SW+t8ESLE/X9zXvpBICkOZ0ITLiHI/Wk7fzH++Om917kgqKH4Zmn0PggqKGztOBYrlp0dAllIRSjQTKw44H5FLQkmSccPwexUO5j91puU/TSKqhLC49uucFC7FqvGHPdTYHwv9nUUlsmQBqkRV++T/II+gIhCGrkAHAWUa1CyNTBCWjiiHeL7kkjY/sJQqoeaOpG78ceA/ryH/Il0hoy05hiVAdArCECYQGNEpjaCDthoFhkVPehrWWnA0kZkwRwLz8nKJzH0CJYCPoWJCBvdyeAQ6wSK4PA5LQbjc3qSKScXMjqRKRqA16CHUnI3Th6JkMb9AniOqAcDPJ8gDAMktZeYCeSmIBUaupCAx008vh9FxTh0xDp/6/fKYrEFtWGRNM2vSramIuf2PNVXx1lpzTBhQKktN735Uv5+P9ZOh1nJpvqV1sXgPo/C51+YA5AYov7yVYYIwsZE8E4aqnVMBix36ORumTEY5fmO7Zhzwt+tdalmipWpNtqW6xi9p1qDyI1CqWesLbzeUcbrgQKTgO3t4/ibY10GMPKdevFwGFb5cmh3lykVp1vImLVXWY9bHeWfWzZtyoZxvli8Ujk5SBdpbX7gCUpuZ+7AUN8LuH87+gKabSCo7Q5Mt45ys6Qasg0q/siRLyIbyDDfNl9vXr/9G/s1N9wXHbg1WzlMbl3IFNN4RQ5+sfDDtR2XCSEKjCMXO6ZZNe9AAFgvlT+2akg+iHSLQ161hdzIjERVu0aosL4I3KwXUgLK/EI5v5BepCHylScrhmrAl+XtrWCdS9uvUkB9f/81Cu7Zy5YTLhz1mUZrNcm5+cdKzAHL7U+fi/Llc2D+Xk/jtul9/Fm/nG7Ja/7LLAxT+Zd1OY90aaS6UkdYWBE0c2XiiPsQcUHAePvyX1UJdRsk/S8tokH1iLamLZMHYMPXFEjL2oL9MQo467S+TpOFH/oXiP+Dcv0xKJj/8vykyD7UALpPIb9UMuDRuDrECrvNAiIY4Z3IZs0HnOkB7w2L43IrufSuZ6UvO6X4bWdALTCZedBLuuVMhh5+Iz4380EPur9xDlSdWTpn8rsmKMekHO0Ql/2D/kzx8KMrIBtbg5T/jcxT2n8H1fILdVqpm4sDHj++Ijunt+OVG8uyUfcIGilE+d4fnCHgDIXyv/Qx5uRv5vGaaJHRHhDRkAVY4Nix2xzjlvGR6a0wfo+8hSAGNZ5jwmHDzoKVUsTDsJFZk7ApZkdFZZCV8mXG+68G3VczAyQHiLAciRA4udmZ4Ri03BUNfOgA8DoMw6rDJB0HeM5F9dSku1pyKNOxADZGRyo+EyZ6UMy9pglCts8RyBj9FNPsD7dB/3L4ZtILPzyCLw4CYhkf5YAPZ1Bq1n20oVvbcOaHYJ4xbnyCSItb+ePNqBXfsoIV9Nohuz/Yai6cGGMYYS3sOPrz60A/Qem8zXG0Fv2egzSwBtQI9T0HNNURB7CEPswd8M1WP29xPqQnOiVly4ihxGdstKCC/Z5BBTIzEzRDDhvX6Np4sJyLnpQvnPDVhtfU660KV6JnWLfQVOg9YoPOuzLSU4Ip4AvacNhOQ8XN53ha2bwtz03YPH2m9BFHrX0xLCN2Aoiuo+jRLqRpSFlwRI60Fah0WiMfs/zOuihOxUy6LI+l869LYNBMtTL7j6WY1tzbKaUhB6+eKCcfel3aZLOqBGmAYJajDT0wHzkE4iJVZn4SIc27zaQXJhS9g3mllHS9EbgZHiBWmqrn1Eol6ePVh2vVYZHo3HTWP4ch9nClrJG7XLFrXSeg+FK8WVMRbFps1yQzj7A9qp0UmlJ96OSP37uOamky5j8goyqzj4mrmypJHTSIuNS59vYoxZwkIo2S6OyaYVIat/HXI9pjjA0Q0H3S+YGbS0F+B1g5sl6wNt4Tx/KmgEq/HeW25SQ3bQC49qZS8cNl/eP0fP7ZWeck41G6+koOihuUwrdrl8k9TlDAXRJ8ppoABQszqVPhtpHX5M5EqtmEcrJ+Buan8xJsFobtNOh8Z4BwVxKyW1N6RL69i2Lyyf739EkRk5z0BFDtGEwp8NT+EQWDAfZ5K1hHpOxgLDmw1LY7d4k0YDUrrCcMGdnwiZAzaSovdo/ibduS8AknBs0r7fqm26OZTc63CLwVwCNOQ72fimlvjCu/2cyzTcN7AsZ1wJLznP90aoPfVKRSiqO0Bc9Q55kXKjVQ5yiqHWJ4Jo6uVghUtUmGUc6dyGpdbyq8efXvn0GTIP+vqx6MhS5k1PePa9jliW38OqL0OeXNTBZy4fVIfXtk24aBxfUqqSSwj3YoGBLhO9mvgvazoQ9/CGfIeiGciasDGHmgCtJvl2QDiTu0BGFLH50Po1N4VAk15ppGnL9suD5c0PkZ9WBfPjpH7sEdu+Be3L8YqYfsnJlbzJY2MVHfWtRuniN9X4BfuJafakISJzEB4D7/4xyUh/YfH2qFwXtxeFNrbANwwbixBfC6ZCMgCiVlRkzCstLBNznMtRVhgpqDo2aSrQ6qOpAk/FKbotJeGW9rZdQU7yrxzQ7RCFL7f2AThibO5HXiuOdz9/aPO5278Zo/YQbDO6NU6/6ys7EOLyq954Qu5iisXHI0laCy8YiLiWVx8OJLCVXksdrk5GdFoDZpQ0ba/FtlyCUqTKw2Fr+pZQyOTUT5rmCEX744NWlhH22H2ehvJWxyt7AgIMdaNWs71WfF7reXgjiBnsEg9QRV+VmTwwRAFXhlqF9lnVohAREAWYLbgb757kcaqhmqsxq9QsCmC/Wl+ksSQgoh1rnk/fHJxskQqIDEYyri+JimqQRKtIXoqfOSKDH/pEAny/D6UZ3d4yz8YzINQHmUcHfkFtctS4UW9TMxph7y/wK+QlAkODAG8SpWMXiWQMLGU121e2B+pqhPi16rg0D0plUqhRNiyProFbjVUsaBt38v+fBDkw6f/JgwJpURnSVMB5jLEBI0wdZCL0AdB/ouJWG71tf8+/N7e2H4VZSEW/utDxaJDvZEhKo70qjky0Ets51Zau7SvQHhLm5qtW+elCpbs6x158X+RrP/XNK/qoRQreThKabZYS4VpwyLtUj5lvtDiqPVPzaU5FCztj3w8s99eEjNUlJ5LraPhMw7vc2nEMik7Cq7MzCxtXRYfgLmGKcqNMBwKIaRW5WamHwETpwPARP/8CmhM11hvdjQM5H0xIKkPOAABHhGjY377IOCIZF3NqX9rWjsbtgmLFD5dwRydvsOs1fzIxmqVQiOP1LDpSkdUzJ8s7AMFK+dm8HZKC7WX+4gK4TwZN3UfwJgpiA7UAMcAdPPyZllOFR86A2cDZmcrgimt0okW7wxQfsbVLaMrDq2CiFOWDF1pRHu+pe5G27vs7gNzWC5ZxEBEuzPqIzd3jpaUGCr6aEbeEi63oKo6iomYRXhpuxQea1lro7LVCi9CGlmM21RiTRa45XweFri5z86CoB5fZysICevZDXALxEvyc9vew/Zf+GAt08UVgnzhRSolv3Rb/NdKsIgJQjmXES5RSc4UnmkPAUfZNvXS0ZpcdVbokol8i2lEp4w0HSxEClxF8vMSkqMgi8y4kEtAnkZSpjOV8uzEh2sfYXIDKpJJwkZvjRiWNOMmVLQxmIYj9ve9m94Vti6lGgfeHlyzqr/ZBB46MFrIKksf8mEr9HZoedJwREK8IH3MbMEagqiiJSjnCxo9TTL1u9yxrrAGa/KTTOMVdp1yZv9liZ1ktzStwiuu/4PZSlVFND7L58eopPn8b6qNDGrv4eR/x+YTy0Yly/l6GIBZj0z8YkK1CX5IQwOZmbPWIDZvZWsQXU0KK+Ge50SoIALWfx/GhcWiJ5j0LkLVMcKxBzLsdEhUgWQgY5iYgVJSnYYtbmjfbsUhYmI1YK3OhUmDiPsRMTGLlbTK+iSImIhkgiXwfu3KS1J+2gEcOyVAmZmV3A+wmphnmlC+pbv2Yfna+lr3VG2twS9i8vOne7KAiGYafPbKmm4KUqlMGb7pbl3TOI/mOksSOqD6pDgsFmDosPPqV38iucs7zv9dcbmgvFDtmJpjZjfw/GHp7N+CyyUX/4KWR9OzYA+PLmYOKtwFzkRTzvb5Xc90WTzldL/d908358zAxHO+Zwb2T8yiZNJVfPdrgNLCAHWtp46yuvwYFavL/8aaXDSmhl5XHyS8rr702HgmkUxrdVHOaFNjpNSsC7pnga8mbOVuUBZPSLZnxAaMIwy9IVU3nmc4dOPO0guVCcHE6kW4rjXteHyxn/z2N4dQnx4x4YEzrg6fsf3VITNGScyZmHiNlxnnxHreVMQ3dngXqjLSrroyLpDgcF/7EjQ8FgI1PVStsgSLhTSkVFF/tgWr8dlKSAVzupAbuCNvXv/wU1jjaVAHbCXXLfywfRRtD11WezoysfIpi3p56NDZQWyGq1n3y/mREgBiw5QUduXIhipGF9zH9oJS4B7QsSo01KmKVpoDkl8UwM+f7q9d2ZJTsh8+kf8Oq4z6W0Vkupj5u8ffbnQKEVuyqBosT8s+h2PD4Z3dZsmorHd3LjnQ+tFUNfK+NrRNsK5nMBqtJ0JbvEFkwbpsg2YiAic9Xl908boJ9PJS+Y3um95eL9YCKS2K3bM0xtPywVQcBc0SxqnyhVHBaf9mZykYWZ0gZjrldFd6CkamucrO22+2Oy2GmdvROfqb4jBsauGH+shV96zy8lbrvkFZ72+5yAxRVHQFPrEw8nW7O0WTxXtaPZMz64VwC+gmYCcTp8TraoP3Lu8eflrtEerqUqKL20bvGHQW0zZ/wStnInbEtVNDx4XU1uUPMrxOx+UDx55Hfedd33n1TMmRUgLytsTex6qye011tcDXVTc3Ks/fYWqIvFtTtQJyZQIXKYqRqTNXcm+OCroCZWchLsGEpc4YcPcuTI7kZfECn4+6uktMTPdLqtL62TLMlskfQbPYbq1PYMgn9gfMGtoiwHcZRVnKXFo6ofYf7jNXH9/++rJ3RaJMKTuhN3qJBpcDu+640d/k1uWdQaNZ1L3b1lQ913bDueMQMZnubJsR9ngOuCHzC+NQfEYqbwvmERV3PFtJQXa7SCPTFadhGWhQ7u482FPa+xNTK0eZgjjq+Gt0CanzQOP4g888zhJmZlouR9d6DBUQuTRulrwiqAd6YT0Fh6x5hZWxIyrIAki0tmZV3LToqCFU7PD87WPFmrac2qlYYYc+FSsqY1tWYKv8BRBF8/dPlJSmwxEObbyDt2Qe0bcbCPHosjWlmwnbomIDODxXqX5yF3QSwPbvrRH9t4reIwrKXEbLmLLnrhtIr1mKJVCtAYUUN5YdfmRkoIbaBMi/WnAB1cJYv70VdyM9EbQBDCZemh7u0cCwkiSxAYujRhOqtYwYhsO2zKzdcWrZHPZhHtD7w+Z74ntDaD7qw70LyvjW0/noOBrSnV8GC45KF3uStqRW/2HWp2OSHT2/HuTlqNknzv9aZwvnT32vXSsb1zlrFMtwtnMwrR27IuNKeLo5FqVZyQuiozXEGQeNPhXFLvLOTqX6qaj88vsoOOZb951cP0thlOTca7atLGK3xVRKX5N3v3xCBfLxc3hQ+3dtqIgdmPwNA74jS8pUOZTXM6mSVl8wKSgPlFUjd7BRgLf+c/cxv3WaL2NxRXILbLU2M/LxcwVGcFwFlHtftAFKg9GVd7WDnnbQHiXlO0b1BUAm+3vaeadNSlZsA8Lankzuq54cVq0VVGhkwH4lTQl8uM/jTk3p2QugQ10cBCG8CezP4yFqo3O0kDrZS2S01DO/YMFCSTK6QG0PqTgProV/Wi1hkZJ5a38sMZRbomCVcarsqdg5lGPJ9zrXE0aiLCvQMlMRaKLXMuMx2iVQVJKO4MnvmTT09Cz53HD1OxnjNjLl4YvBCClXk7S6R1Um8v0pBfi9Sa6oJjEsmTP7urlcFY6uDgoh7qGrdmrevRVYi7cC5aMbGCDx4SewCq/YSIinqvA6B611H82NxhpbZ5W8QD5Z7LVjNyfTzDPFmd95seYbYoWerdZVa3Qve5W54P1a7Mtu/nbsVwzDjN2oysxUJtDVugRmYBhfihVog9YHE5nMtN9znQMz0XBR6pt4TTfQxbWBbHINdxyMU7OprHv3qgbLZTeUa1Q6tQ1jN0VdxXQrN7u1kRXAabr/ckabdLNW0hgO8dmZYGVFd63qwrUZ8djIFRLJ9HXnuPm1iK1LYFvdnhffmTXsPIO+rmmG/RjxAbPlXr1UUXdWqmsr5OIBTBE8C4eq/ybHxcmP0CJinrd9dx3Zr5ggggpZa2Xvd1qxHj0GRmidhvlMNNoTBR7kN3kvKO+uHKjdyn/+sqaLn2e2pn0m+jxWY/WN4oqg19pnWRVQ9Z975H3UHnelSNPQWtBSBV8Ax1KQRMZdty3C+HzO+nwIr1xq+uUYqCmocISF7C+Qyn9qhVLHS1ZBZUt7FmRLQYBGa/xoQ8L2HN9M94vY3iQ0Gac9/eVUHxZ2NbB/KdATKdDxijKBZIYZtM7cMhmyQ/syiyMIr7Z69Mm9xa4z/FW+ujSa4IR+vRyi11BEBau9/6am3OW7LpHqMvTiDpl67zpLbV4XbP327uOTRmt46Us02vFqTPRULPdMDz0fLPeWlPHs9PGUeq7Xey6NmhOX9rtqrOlLsm0VEJc/CrCJ0nCC9fbSdMMa8p5+eXFITVFgN0FsiORaoPs91DnelHurYNal6pVWpY49jNvMqjFljyFxLLMuXhXlkSQvcE2m4WLvj5NMrYD09pJUUHOz4YJ2jnjVWnVUViOV0tOl2iu+GPZkZsvT5dstTRb0mi+do47nzMUrE7ls8GefguiOEh6iOJ4u03RprtuUtosde26i9CJVRU1H2EPm87vHsu9xq7J1DKGXqhqqOqFJcUBH9ATHDtOeyKZvQU94ZjX51FIYfVw62NIouHWZSqO5kN3JqvH2hQtYus7gcypkuCXLYAZMKCtvhRS7RGa6tEBdB1spiO9kzoFqc6MgAmH47gZ329X7j791M4gzbWpXbpN0qcmVXieQvAyV2Q9nnvXSz8y8XxiHmwWNnsri9JI57z/+VpB7AFXI6zPT82gPCJx46jVaM1BURWsWUT53rJpflmqsho0LTyyH7a2novFCRU843deduZ2EXXp7mdwqPbLBfOscss7Pw/iWv7Hw7WjS4lWIqrqo7bxuB7e5Iw/i1DOozW5OhRVqkEcHSEeCTfsui+JP/u1wR+2Ng0j8/+Gjnt2qeFqdg83ase/j2YtkrI6gxe2KundqFFutQEFsP7EvAIbQR8rDv6SafwN0I9AewsmLX+2nXrj/1GRtRUiUd1d8MMC9u8J3eIfFyH3ur3u2Bpti4OWamFVvdwyUKD3vDLucoPAMe2Laf2L1may81sTcpby8Q9MBdHT1+jw1ITKrOGnHkrLvTu8gUs5xLPrUm90higqdUsy7FE3IX14TIbvjvtMarkrruZ35Yrj2z0YXTbkktGBkkF/jSme2NL0YWj8VaY8DVy8TsGGRwee7LoWoXyvh2IgKIY27q+AfZhhEaU7lgj+xkA4fUS7zM5dRtW/vX1UyU1fJHFAk46oJL0Vim2/NO8WDumYJSrlon3s00j/Tv0Chiu3m65yc7EnWjGITk+epuiwZ8PDqQ97ZVAos87fcdhVylvzDCcc6bCiv1vvaY2zoITmLdu0GqvkN7UADVWY43JFHb19+GthhdQ9T/BC1Ht+Vi9Ggi/YjwYeGT/7gb19nuMY6lrARLtMtvJUbJ46wiZBUXj+o92sZhIXFrYKk44HYQUeh0BwgPQVL8oHHoTFTNlGugHHjjsLyh0wWbPoVcsOOQhIDnZ4lMT6oF0ZBHsz3mmxA7UgmOHsC7k0dZtytdOuWUoVvfTBBtEz8XTrKiWYm8yqVGZLQnXdiw6Rl4knIbdO5PJ66krDKtZG1e4AOWwrzWHzvbTajGGys3lfWJfOI2ipa0Zp1NFrtNr5/8vcS+nhFk6KfnzvqurYkNa3beUfMmzfpvnFLMQABhw2ED4+DO4vatXDj1gGEObAT0dzClmE5PQjFO1+IaAcnbvBrwhyWj28f7glViu7cvco4EzEVJtx7Pmb6KU+fTbSNqu8TuZitm2TP/Kc84HGGyiLhXQamjXWb92FCH3p6luCwcT9LlpTxyY6yyvxu3P75cX/pMc3Rj+/aSxKKTXsU3SIKp2/DrdvRvZhWciphFRy8KjRryWMMw5Pb129+uLHuTw5hHzy7P09gkHh83sD2EF0oWeGNMDtvD9pCP4Fq6K7JHl2ough5nxc3mz7DoYUVHpVjqk1o5ZCQNJ4f1Wn+M17/pjGpHUz75jx6uvwsHDFltjieSp0tbsYROcdGt8E5A31OWxNipsTQJM0nxG653hbDPmwz3ykph4LBcXf2UBHn/tW1s1Ht/4wmWdru0lZ0K/8K0TyS8VF8+vTwv9/9n/f3xI5TtibzCL/XrvFi+1GIismY3/QPouhtmeZ3XNFtrNWtK7xy/c3GojTzpX/B25XDO9iVXbXr9w07Z/YJkP0FlsPnrxVFtjLozcmxDG6GGZejZq1UnWFh3fCFab2Y04ljUNg3f/gml7PO2wWDYr/nrrVwEciOzGIFVfi1rtPhyh9Z6kHW/aTZYGjBafueI+x6k+6wWX3+qTJn276QFtgxru/nd49+FF0aOU69HxdXdA9adDlm3Q9j+I0z6/p+14MYQRBLmrBWr7ihCOznjpmcy4jyGQs35Wz9ungk5/Y/3sxez97MbolU5M3r17d3r+9//unu7c//eX/30z/+/uPd3e040/a9xUEeHgmNY+V7jbKimR8V5OFx84Od7OFx82PxoSG0pVI1N0SniBf0vXlzCHw7VQ8mBYk0cAEM/4hAJua4p+4sLPcEDOf5Wuowqp73Qv/9x5s3t7c3t7f/fvP3H2diO/N/mUWy9dp4D+bHzx+JgkiqOHjoq3xNZuQBX9OTC0OxS9uGUaJgA0q3j+eHR8KlfOpMmDXYAIbH85Rnei5HPblUvp96KPn4Js9yCZFPlKY3LoQWS7SEr+Dz+/uXuYnveWEXzVWYSgEkke1rSpwugNfe8LrGAexo//MWXc8XSylnC6pmK8mpWM2kWs1eWP6+qP6ilfQungOyY8RgQCVM5G++2OFJJBPwXYepIJAsII4hJpFMd0VgkJpWmyH8wtqY9O7VqzRbcBbpbLlkXxHHYFme40uYhzoobeH8Tzuc/9AiJ9O1lyrWBCXQixvxFzV6EHc/f9Z3xo1/OG0vAP+wzIEgAoGIw1BM/dbZL5V3zkht6L044Ouhz/hZ3zjDcppj+IHtg0aLRPhb4yfuDCv1TL3MOJ+PEIW6Ddydnv+EfydD3z8dkZ2XS9emP7efWZmT9wGCoyzodkvSg/u5v0U5FsJZ1M1F6I1J7HXLfafQPoc4XP1hgSEPu9FVe/trA4EigQmxFFOg8dP5XuyU6+IejD10bXo6OnUzZIKnQ36tXwyvupJ5wOe6bLddhmaKZqS+DhfLU11ALXXPwP0BM/JOKgU6xcZrRub9pjRgXvuV1Ziv9E6/EmBesXTzwysTpfMEkhn50NH2v7vML9z89+hO7P2rSwYGgKRK13R/nXf3Sg9Ei4jdXveL5KeF2Ip8vrTd/N1LQZcOmZqAXJ/0832YXjkBPgttn55pwgNtLQKm161k1wkAlnmwyrSjuBlxqWG+pZ2tQ06CtoHQ6oh5iWQeTAjVcRuWXAbsAsgQ1Hon5jr8oNVZQec4hmJWEDUfrX0WzBbHEMxLJnBNmqGgs4MugIxB3Yz/PBvqN0NQc6rNnEahDMxZQec4hmC2uuYsJ0i/ymNiFUJcOGnxpObrb/d/EvPVEvKM5msWX6L5un91yUDz9dzGXxfqPf9S7I608YjF6CjBFzfEl3o3A3+dQaxyUXGf8rGEI1NtvjH7LAlXMwRSA/n2yb/a+DMTaWbm+YcSxjkLlw8MKOj88CmnFV92KIdql0tlGpTu5f0BxVLv5WoF8U3x9jlozaRoBpD38bgjnHZwmWt5CcuDCc6qofV41BHzvhXV1AiXK2Y1V3OKPfe9jqT5/udM+0pG98raAA4EkrBHorBfz2euSkPHAoRqRY5Zg0L4hpam1NMTQSQLKTm04gO9SOzXCBMxi5xmonlmaC9HjilxC69I3vm1Ufi2B0Mkp5aKymo4BR0HZinL3mncOqwOrqleA77rSR6H6QS3RvORKdfeI/RtLS3oc9Jly9QGoPJf/n8AAAD//y766bE=" + return "eJzsff+PGzey5+/5KwgfFhm/m5E93mzevvnhAMfzcjeAEw9sB/uAw0GmuksS37DJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/9YXFqmKxeEOeYHdH9E4bSL4jxDDD4Y68+IS/ePEdITHoSLHUMCnuyP/6jhBC3B+JNtRkmiRgFIv0NeHsCci7x98IFTFJIJFqRzJNV3BNzJoaQhWQSHIOkYGYLJVMiFkDkSkoaphYeRSz7wjRa6nMPJJiyVZ3xKgMviNEAQeq4Y6s6HeELBnwWN8hoBsiaAIVMuyP2aX2s0pmqf9NgBT788V97QuJpDCUCU24jCj3o+X0zfznq/NW546kguKXodn3IKiguLHjVKBYfnoEZCkVoUQzseKA8xG5JJQkGTcMv1fhYP5TZ1r+0ySiSgiLa7/OSeFSrBp/2EON/bHQ31lUIksWoEpUtU/+D/IIKgJh6Ap0EFCmQc3SyARh6YhyiOdLLmnzA0upEmruSOrGHwf+8xryL9IVMtqSY1gCRKcgDGECgRGd0gg6aKtRYFj0pKdhrQVHE5kJcyQwry+XyNwnUAL4GComZHAvh0egEyyCy+OwFITL7U2qmFTM7EiqZARagx5Czdk4fShKFvML5DmiGgD8fIo8AJDcUmYukJeCWGDkSgoSM/30chgd57QR4/Cp3y+PyRrUhkXWNbMu3ZqKmNv/WFMVb603x4QBpbLU9K5H9fv5WD8Zai2X5luSi8V7GIXPLZsDkBug/PIkwwRhYiN5JgxVO2cCFjuMczZMmYxy/MZ2zTjgb9e71LJES9WabEt1jV/SrEHlW6BUs9YX3m4o43TBgUjBd3bz/E2wr4MYeU67eLkMKmK5NDsqlIvSrBVNWqpsxKyPi85smDeloFxslgsKRyepAu29L5SA1GbmPizFjbDrh7M/oBkmksrK0GTLOCdrugEboNKvLMkSsqE8w0Xz5fb167+Rf3PTfcGxW4OV89TGpVwBjXfE0CerH0z7UZkwktAoQrVztmXTHjSAxUL5U4em5INopwj0dWvYncxIRIUTWpXlRfJmpYAaUPYXwvGN/CwVga80STlcE7Ykf28N61TKfp0a8uPrv1lo11avnHL5tMcsSrNZzs0vTnsWQG7/2SmcP1cI++cKEr/d8OvPEu18Q17rX355gMK/vNtpvFsjzYUy0vqCoIkjG3fUh5gDKs7Dh39ZK9TllPxaekaD/BPrSV0kC8amqS+WkLEb/WUSctRuf5kkDd/yLxT/Afv+ZVIy+eb/TZF5qAdwmUR+q27ApXFziBdwnSdCNMQ5k8ucDQbXAdobHsPnVnbvWzmZvuQz3W/jFPQCDxMv+hDuuY9CDt8Rnxv5oZvcX2cPVZ5YPWXyuyYrxhw/2CEq5w/2P8nDh6KMbGANXv4z/ozC/jMozyfYbaVqHhz4/PEd0TG9HS9uJM9O2adsoBjlc7d5joA3EML32s+Ql7uRz2umSUJ3REhDFmCVY8Nit41Tzkumt8b0OfoeghTQeIYHHhMuHvSUKh6GncSqjJWQVRmdRVbDlxnnux58W8UMnBwgznIgQuTgYmeGn6jlrmDoSweAx2EQRh02+SDIeyayr+6IizWnIg0/UENkpPIj4WFPypnXNEGo1lliOYOfIpr9gX7oP27fDJLg8zPI4jAgpuFRPthANrVG7WcbqpXdd06o9gnjNiaIpIi13968WcEVO0iwzwbRrdleZ/HUAMMYY2n3wYdXH/oB2uhthtJW8HsG2swSUCvQ8xTUXEMUxB6KMHvAN4/qcZn7KTXBOfGUnDhK3IntFhSQ3zPIICZG4mKIYcN6YxtPllOR89KFc56asJq8ziqoEj3TuoW+QucBAjqvZKalBCXiCdiz20xAxk/lflv4vi3MTd89vKX1EkRtfDEtIXQDiq6gGtMspWpoWVAiRloP1AYsEI9Z/2eUilOxU4rFkXQ+uTQWzUSCyVc83azm1kc5DSno/Vwx4dj70orJoh5oAYZRgjb8xHTgHISDWJn1SYg45zKfVpFc+gLmnV7W8UrkZnCEWGWqulsvkaiHVx+mlcci07vpqHkMZ+7jTFkncbtm0bpOQvemeLWgIt6y2KxJZhhnf1A7LTKh/NTLGbl3H9fUZMp9REZRZgMXVzNXljxqEnGpUfT1KsacJSCMkunumGRSmbby1yHbY45PENF80PmCmUlTfwVaO7AVWRtuCeP5j4JKvB7nteUmNWwDufakUvIiZP/h9X/82JLyknGo3XwlB2UNy2Fatcvln6YoYS6IPlNOAROEeKpT4beRNuTPRKrYhnGwcQaeTeU73iwI3S3S+cgE56gkZrWk9o58eRXD5pX96+2XICI77wmg2DGaUOCr+SEMAhPu81SyjkzfwVhwYGtpcewWb8JoUFtPmDaw4xMhY9BWW+waxd+0M+cVSAqeVdv3a7VFN5+aaxV+KYBDmIZ8PxPXnIwrvNvPsUzDeRPHdsKR8J5/d2uA3lenUKiithvMUfuYVyk3UmUrq2xi+UkYXa0UrGhxFEY5dyancbml/OrRt3cOPQz5tW5+PBqylFkzMq4tnyOW9eeA2dMz8qvEgoR/MRHLbYf+uakDQd2+VRCWdJsRoFFeJRdILCPdyg4EpED2W+S9rOlD38IZiiaIZypaxMaaaAK0i+fZAOLK7QEYMs/nQ+jM4BUCTXmmkacv2yEQlzQ+xpzYkM+Okce0RxqAF7cvxhpl+ycmVvMljYxUdzbUG2eY31fgF+Emp9qQhInMQHgNv/jHJSH9h8faYXBe3F4U2tsA3DBuLEl8Lp0I6AKJWVGjMKzUsE3Oc4kirDBTUPRs2tWhVUfShB8KU3TaS8Qt6+y6hB3l7rkhWikL339sgnTF2cIQ3Ncc7v5+UucLP36zW+wgWGeMcl28Vlb6oUflZV7ERq4CyyVLYwkaC7GYiHgWFx+OpHBVH4td7k5GNFqDJlS0/a9FtlyC0uRKQxG7etbQyGSUzxpuyMWHZ4ME62g7zF9vI3mLo5UdAiHGOlLLuT4vfq+3HFwR5AweqSeows+KDj4YosAbQ+0y/cwqEYgIyALMFvxNeK/SWOVQzd14CQWbJNif5idJDCmIWOeW98MnlzdLpAISg6GM62uSohkk0RqipyJmrujwlw6VIM8fQ3l2h5f8g8FzEcqjjGNgv6BWLBVe1MvGnHXI+w38Akl54IEpgVepktGrBBImlvK6zQv7I1V1QvxaFRyGJ6VRKYwIW9ZHt8CthSoE2o697M8HQT58+i/CkFBKdJY0DWCuQ0zQCI8SchX6UMTt1/778Ht7YXspykIt/NeHqkWHeSNDTBzpNXNkYJTYPmtprdK+guEtbVq2bpuXKliyr3fkxf9Fsv5f072qp1as5uEopdtiPRWmDYu0OwIqzw8tjlo/1VybQ8nT/szHM8ftJTFDVem5zDo6PuPwPpdFLA9pR8GVmZmlrcvjAzDXMEW5E4ZDIYTUmtzM9CNg4nQAmOifXwGN6Rrrz46GgbwvBiT1AQcgwC1idM5vHwQckayrZ+zfmtXOhi3C4kifrmCOQd9h3mq+ZWP1SmGRR1rYdKUjKuZPFvaBipVzM3hbpYXa631EhXCRjJu6D2DMFEQHWoBjALp5ebNMp4oPg4GzAbOzFcmUVilFi3cGKD+jdMvsikOrIOKUJUMljWjPJ+putL1idx+Yw3LJIgYi2p3RHrm5c7SkxFCxRzPylnC5BVW1UUzELMJL3KXyWM9aG5WtVngx0shi3KYRa7LAifN5WODmPjsLgnZ8na0gpKxnd8AtEK/Jz+17D1t/4Y21PD6uEOQLMVIp+aX74r9UkkVMEMq5jFBEJTlTRKY9BBzl29RLSWt61VmxSyaKLaZRnTLTdLASKXAVys9LSI6CLDLjUi4BfRpJmc5UyrMTb659hMkNqEgmCRu9NGJY0oybUNHGYBqOWN/3bnpX6LqUahx4u3HNqvFmE3how2ghq4g+FMNW6O2w8qQRiIR4QfqY2YI1BFHFSlDOFzR6mmTqd3lgXWEN1ugnmcYr7TrlzP7LEjvLbmlahVe0AwCzlaqKaPwpnx+jcsznf1NtbFB7Hyf/OzajWDYqWc7X0wDMeuTBLx6oNsEPaXAgM3PWmsTmLW0NoqtpYSXd85wIFUTA+u/HuLRY9AST3k2oBkY49kCGnQ6JKpAMZAwTM1BKqtOwxQ3t2684REysBsjqXJg0iLgfEROzWElrrE+CiIlIJlgS72VXXpry0w7g2CkBysys5H6A1YN5pgnlW7prb5avbax1T9XWOvwiJj99uicLiGimwZ9eWddNQSqVKdM33a1sGvvRXGdJQgdUnxSbxQIMHbZf/eJ3JHeZx8W/Ky4XlBemHY/mmNkN3H9YOvu3oLjk4r+hFdH0COzh0eXMQYW7wploytk+v+uZLounnO63+/7p5pwZmHjO98zA/olZlEwqxXe/BCgtHFDXiuoor8uPUfG6/G+sy0Vjauh19YHC6+rLj41nE8m0XhfljDYtRkrNuqB7FvhqwlbuRmXxpGR7RmzIOMLRG1J143mGQzfuML1QmRBMrF6E61rTjscY+8lvf3MI9ekREx444+rwGdtfHTJjlMSciYllvMw4JzbypiK+scO7VJWRVurKuESCw33tS9BwWwjU9FC1yhIsFtKQUkX93hasxmcrIRXM6UJu4I68ef3DP8MWT4M6YCm57uGHraNoe6hY7e7IxMofWdTLQ4fODmIz3My6X86P1AAQG6aksJIjG6oYXXCf2wtqgXtQx5rQUOcqWmkWSH5WAD99ur92ZUvOyH74RP4rbDLqbxeR6XLm7x5/u9EpRGzJomqyPC37Ho5Nh3d2nyWjTr27z5IDrSBN1SLva0vbBOt6CKPTeiK0xZtEFqw7bdBMROC0x9uLLl43gV7eUX6jG6f31wtZIKVFsXuWxrhbPphKoKBZwjhVvjAqOO3f7CwFI6sTxEynnO7KSMHINDfZeTvOdufFMHM7Okl/UxyGTS39UB+5Gp5VXuJq3Tco6/0tF5khioquxCcWRr5ud6tosnhP62dyZrsQbgndBOx04pR4XW3wXvHu4ae1HqEuLyW6uO30jkFnMW3zF71yJmKHXDs1dFxQbV3+IMPrdNx54Nj9qG+/69uvnulwpNSAvE2xj7Gq7F5TXS3wddXNjcrzd3g0RN6tqVoBuTKBixTFyNS5K3k0RwVdgbKzEHfAhKXOmHD3IUyO5GXxIp/PurpLTEz3a6rS+tlOmC2TP4JmsV1an8CQT+wPmDWsRYDvMoqylLlj6YTaf7jPXH18+8vLXolEmVJ2Qu/0Eg3uDOy6UtL+a7UmvPPudZOPl7c7jWfeeMq3z6hBjuZ/eUlOp0K5/kjB91QObS9A5tNIfRi9ek3VcxljnDsOkZbp4Su2OLINR8oH3Kz6mXEoPiOVjyHyTJxz66yFQbG4DDXTFaTLQKN7d1fGenc+Dp16U5UpiKPcpka3mToPNI4/2FfiLGFmpuVydI3QUNWRS+NmySvJeqAXXndwyFo2oTJ2RAVZAInW1h2Pm5EANYSKHfptfaxY01YyZCpW2KFPxYrK2JYV+OTCAoii+Ts6SkrTkUAJLbyDl2R+EmQXEOLRZYtTNxO218VGguiPUf3kLnYlgM8ItEb03yp62Cgoz8BaTrj119xAes1SLJ1rDSikuLHs8CMjAzXUJkD+1SwamoWx+Z5Wvpb0ZF4HMJh4bXq4R8fUapLERj6OGk2o1jJimEbdMrN2bphlczj2fcCsATZxFN8bQvNRH+5dMs+3MM9Hx9GQ7vwSYXBUuthz2E9qdUNmfTom2dHza2Vej5r9Bv2vdbZwcfj32rVEch3YRrEMZzsH09o5TzKu9KubY1GalbwgOlpDnHHQGItTfI3AxTdUPxUVg34dBcd8676T22cpjJKce8u2lUXOv5hK6Wvy7udPaEA+fg4Pav+uDRWxA5O/hcF3ZEmZKofydiZV0toLJgXlgXJ85A42mPBRY552yG8r52IsrtZuga3WZkY+fq7ACI6rgHKfw2iA0mB05X32YIYm6LmS8j2sugCQyf5+f96xlZIV24CwPiqT+6puh1X5BQ0aGbBeSVMDH+7zfGVTe/YC6DAXB0Ho8ukJeTzEbHSOFjIne4mMlnrmBRYssCWjCxv3kIrzoCz8E30Ji5TMn4jA0lS5JQpWGafK7oqdQzmWfK9zO2Ek6rICLTMVgSZ6LTMeo18CRQXyCJ78nklDT8+Sz40UUSdj3EKmPHyhHCHlZpJW16jKRL4+pQC/NskV1SSGJXNuXzeXq8rR1XkjxD0M4k7Nu7cCazhXoHxWDBNrPm0J1uAVCwnxVA1e56C1Lra501hj66xynpRPFnvr2M3JNPNMce53XuT7hlilZ6t11Rvdy15lLni9Fuuym78d6xXTd2MXqjIzlQkMtS6BGXj8I8UKtEHvg4lMZtqvuc6BmWiEKPVFvKYb6OLaQDa5Rk0OxqnZVN6X8KYGy6w3lGs0OrUFYxdF3cR0Gze7tJEVwGm6/1JPm3SzVtIYDvHZmWB1RXdJdeHa03hs5AqJZPq6c9z8Os3WFT5Y254XbZo17DyDvq5phn088SG85V67VDF3VqtrEnL5AKYI7oVDzX+T4+LkW2hx0pI/H+A6+18xQQQVsvYkgl9phTx6HIyQnIbFTDTaky8eFDf5KCjv0h2o+ct//vKmi59n9qZ9BcN5vMbqW9cVRa+1XbMmoBo/9+j7qDXuStimobWgpQq+AI4lRImMu27phPH5WofzIbxyJQ0vx0BNQYUzLGR/YV3+UyuwO16zCipb1rMgWwoCNFrjRxsatmf7ZrpfxfYWL5Bx1tNfavZpYVc7/ZcBPZEBHW8oE0hmeILWeaJMhqzQvjPHEYRXW4T6w73FrjP9Vb7eNZrghH69HKLXUGQFqz0jp6bcnXddItVl6sVtMvWeh5bavJ7cxu3d2yeN1vDSl/a089V40FPx3DM9dH+w3FtSxrPT51PqZ70+cmnUKrljv6uGTF+SbavwvPxRgM23hhOst5dmG9aQ94LMi4pqhgK7UGIjLdc636+hzvGmXFsFsy7VrrTKc+xm3GZWjSl7HIljmXXxpijPJHmFazINhb0/TzK1AdLbSzJBzcWGAu0c8aoldTRWI43S06X6K76I+mRuy9Pl+y1NFvS6L52jjufMxRsTuWzwZ5+B6M4SHmI4ni7TdWnKbUrfxY49N1F6kaaiZiPsJvP53WPZL7v15tkYQi/VNFRtQpPigI3oSY4dZj2RTd+CnfDMavKpZTD6uHSwp1Fw6zKNRlOQ3YdV4/0Ll7B0HeXnVMhwK5/BDJhQV94KKXaJzHTpgbrOx1IQ3wGfA9XmRkEEwvDdDa62q/cff+tmEGfa1K5qJ+lSkyu9TiB5eT3WGNWYZ6P0MzPvZ8bhZkGjp7JsvWTO+4+/FeQeQBXy+sz0PNoNAieeWkZrBoqqaM0iyueOVfPLMo3VtHERieWwvfdUNOyo2Aln+7pPbidhl95eJrfKiGww3zqHrPPzML7lb3N8O5a0eE2kai5qK687wG2uyIM49Qxms5tTYYMa5NEB2pFgs8fLoviTf4PeUXvjIBL/f/g4bLcpntbmYJN/7Bd69iIZayNocbuiHp0axVYrUBDbT+xLgCH0kfrw31LNvwG6EWgP4eTFL/ZTL9x/arK2KiTKuys+GeDe6+E7vMNi5L7w1z13hM1U8HJNzKq3OwZqlJ53pl1OUHiGvVTtP7H6TFZe+WLuUl7e2esAOrp6xJ6aEJlVgrRjSdl3F3wQKefYFv3Rm10higqdUjx3KZrXv7wmQnbnfad1XJXWczvzxXDt10b3VbkktGBkkF/jSme2NL0YWj8Vxx4HSi8TsGGRwWffLoWoXyrp2IgKIY27q+Af9BhEaU7lgj+xkA0fUS7zE5dRtd/zX1UyU1fJHFAk46oJL0VjXRa52fYZbc0SlHLZPvfYqEywLGyBShXbxdc5OdlzWDOKTUyep+qyZMDDqw95R1wpsMzfcttVyFnyDycc67ChvFrva4+xEYzkLNq1G+/mN7QDjXeZ4XBHHr1/+WlgZ949TPFD1HrDVy5Ggy7a1gQfqD75Q9F9HQUbcixhI1ymW3grN04cYRMhqbyaUe/zMwgLi1sFSccDsYOOQqE5QHoKluQDj0Njpmy+XQHjxh2F5Q+ZLNj0EnLDjkISA52eJTE+xBhGQR7M95psQO1IJjh7Au5dHWbcrXQbllKFb8QwQbRM/F06yolmJvMmlRmS0J0PYsOkZeJJyG0zuDyeupKwyrWRtXu4EFtR81h87302oxhsrN1XNiTziNomWtGadzTa7Da+f/J3Nvp4RZOiD6Tb6rqWJDWt23lHzJs3d79xohiAgMMGwpvHwR1prSzcuHUAYQ7sRDS3sGVYTw9C8c4XItrBiRv8mjCH5ePbh3tClaI7d68yzkRMhQm/WRAz/ZQfn020jKrvWrmcrZtkz/yn3OBxhoqQ8C4D08aGzfswYQw9PUtw2LifJUvK+GRbWWV+N27//Li+9Jim+sd3eyYJxaY9im4RhbO34Zb/GF5MqzmVtAoOXlWateQxpuHJ7es3P9zY8CeHsA+eXZ8ncEg8Pu9ge4gulazwRpidtwdtYZ9ANWzXZI91VEOEvM+Lm02fYdPCCo/KNtUmtLJJSBrPj3qh4DNe/6YxqW1M++Y8erp8LxwxZbY4nkqdLW7GETnHBsnBOQP9cVsT4kmJoUmaT4hdlr0vhn3YZr5TUg4Fk+Nu76EizuOra+ej2v8ZTbK03aWt6HL/FaJ5JOOj+PTp4X+/+z/v74kdp2xN5hF+r12DxvZjIhWXMb/pH0TR2zLNr7ii21irW1dYcv3NxqI086V/wduVwzvYld3Y6/cNO2f2ByD7CyyHz18rimydoDcnxzK4GZ64HDVrpeoMC+uGC6b10lInjkFp3/zBpFzPOm8XDMr9nrvWwmUgO04WK6jCr7ydDlf+OFcPsu6n8AZDC07b94xl11uGh83qz58qc7b9C2mBHRP6fn736EfRpZPjzPtxeUX3EEpXYNb9oIpfOLOu73c9pBIEsaQJa/WKG4rAfu6YybmMKJ+xcFPO1q+Lx5Vu/+PN7PXszeyWSEXevH59e/f6/qd/3r396T/v7/75j7//eHd3O861fW9xkIdHQuNY+V6jrGjmRwV5eNz8YCd7eNz8WHxoCG2pVM0F0aniBX1v3hwC307Vg0lBIg1cAMM/IpCJOe6pOwvLPQHDeb6WOoyq553Zf//x5s3t7c3t7b/f/P3HmdjO/F9mkWy9Ut+D+fHzR6IgkioObvoql8mMPOArjHJhKHZp2zBKFGxA6fb2/PBIuJRPnQdmDTaA4fE85Zmey1FPdZXv7h5KPr7ltFxC5A9K0xuXQoslesJX8Pn9/cvcxfe8sEJzFaZSAElk+5oSpwvgtbffrnEAO9r/vMXQ88VSytmCqtlKcipWM6lWsxeWvy+qv2gdehfPSNkxYjCgEibyt4Ls8CSSCfiuw1QQSBYQxxCTSKa7IjFITavNEH5hbUx69+pVmi04i3S2XLKviGOwLs/xBdVDA5S2cv6nHc5/aJGT6dpLFTJBDfTqRvxFjR7E3c/m9e1x4x/c2wvAP0h0IIhAIuIwFFO/kfdz5X08Uht6Lw74eujzjzY2zrCc5hh+YPug0SoR/tb4iTvTSj1TLzPO5yNUoe4Ddx/Pf8K/k6Hv5o44nZdL16Y/959ZeSbvEwRHedDtlqQH93N/i3oshPOom0LozUnsDct9p9C+gDhc/WGBIQ+70VV7+2sDgSKBCbEUU6Dz0/nO8JRycQ8NHyqbno5O3QyZ4FGRX+oXw6uhZJ7wuS7bbZepmaIZqa/DxfJUl1BL3fOBf8CMvJNKgU6x8ZqReb8pDXiu/cpazFd6p18JMK9YuvnhlYnSeQLJjHzoaPvfXeYXbv57dCf2fumSgQkgqdI13V/n3S3pgWgRsVvrXkh+Woityuei7ebvXgq6bMjUBOT2pJ/vw+zKCfBZaPvsTBMeaOsRML1uHXadAGB5DlaZdhQ3Iy41zLe0s3XISdA2EFobMS+RzIMHQnXchiWXAbsAMgS13om5Dj99dVbQOY6hmBVEzceOnwWzxTEE85IJlEkzFXR20AWQMaib+Z9nQ/1mCGpOtZnTKHQCc1bQOY4hmK2tOcsO0m/ymFiFEBdBWjyp+/rb/Z/EfbWEPKP7msWX6L7uly4Z6L6e2/nrQr3nX4rVkTYesRidJfjihvhS72bgrzOIVa4q7lM+l3DkUZtvzD5LwtUMgaOBfPnkX238mYk0M/P8QwnjnIXLBwYUdH74lNOKLzuUQ7XLpTINSvfy/oBiqfdytYL4pngzH7RmUjQTyPt43JFOO7jMtbyE5cEEZ9XQejzqiHnfiurRCJcrZi1Xc4o9972OpPn+p0z7Skb3ytoADgQOYY9EYb+ez1zVhg4BhGpFjpFBoXxDS1PqxxNBJAspObTyA71I7NcIEzGLnGWi+cnQXo4cU+IWlkje+bVR+LYHQySn1oqKNJyBjgOzlGXvNG5tVgfXVK8B3/Ukj8NsgpPRfOSRa+8W+rZ2LOjPpMuWqQ1A5b/8/wAAAP//AKSNsw==" } diff --git a/metricbeat/module/system/filesystem/_meta/data.json b/metricbeat/module/system/filesystem/_meta/data.json index 2cb9f0897ac..4f13fff0465 100644 --- a/metricbeat/module/system/filesystem/_meta/data.json +++ b/metricbeat/module/system/filesystem/_meta/data.json @@ -1,33 +1,30 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.filesystem", "duration": 115000, "module": "system" }, "metricset": { - "name": "filesystem" + "name": "filesystem", + "period": 10000 }, "service": { "type": "system" }, "system": { "filesystem": { - "available": 53067177984, - "device_name": "/dev/dm-5", - "files": 0, - "free": 54342492160, - "free_files": 0, - "mount_point": "/var/lib/lxd/storage-pools/default", - "total": 86301999104, - "type": "btrfs", + "available": 148708327424, + "device_name": "/dev/mapper/fedora-root", + "files": 105089024, + "free": 148708327424, + "free_files": 103974920, + "mount_point": "/", + "total": 215211835392, + "type": "xfs", "used": { - "bytes": 31959506944, - "pct": 0.3759 + "bytes": 66503507968, + "pct": 0.309 } } } diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go index d64ef95f562..9238df79f59 100644 --- a/metricbeat/module/system/filesystem/helper.go +++ b/metricbeat/module/system/filesystem/helper.go @@ -145,20 +145,23 @@ func AddFileSystemUsedPercentage(f *FSStat) { // GetFilesystemEvent turns a stat struct into a MapStr func GetFilesystemEvent(fsStat *FSStat) common.MapStr { - return common.MapStr{ + evt := common.MapStr{ "type": fsStat.SysTypeName, "device_name": fsStat.DevName, "mount_point": fsStat.Mount, "total": fsStat.Total, - "free": fsStat.Free, "available": fsStat.Avail, - "files": fsStat.Files, - "free_files": fsStat.FreeFiles, + "free": fsStat.Free, "used": common.MapStr{ "pct": fsStat.UsedPercent, "bytes": fsStat.Used, }, } + if runtime.GOOS != "windows" { + evt.Put("files", fsStat.Files) + evt.Put("free_files", fsStat.FreeFiles) + } + return evt } // Predicate is a function predicate for use with filesystems. It returns true diff --git a/metricbeat/module/system/fsstat/_meta/fields.yml b/metricbeat/module/system/fsstat/_meta/fields.yml index d9d2b0e7ce1..d9bd3acd7f3 100644 --- a/metricbeat/module/system/fsstat/_meta/fields.yml +++ b/metricbeat/module/system/fsstat/_meta/fields.yml @@ -10,7 +10,7 @@ description: Number of file systems found. - name: total_files type: long - description: Total number of files. + description: Total number of files. Not on Windows. - name: total_size format: bytes type: group diff --git a/metricbeat/module/system/fsstat/fsstat.go b/metricbeat/module/system/fsstat/fsstat.go index a8a1eaa0c94..0f43cf4af55 100644 --- a/metricbeat/module/system/fsstat/fsstat.go +++ b/metricbeat/module/system/fsstat/fsstat.go @@ -20,6 +20,7 @@ package fsstat import ( + "runtime" "strings" "github.com/elastic/beats/v7/libbeat/common" @@ -91,16 +92,23 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { totalSizeUsed += stat.Used } - r.Event(mb.Event{ - MetricSetFields: common.MapStr{ - "total_size": common.MapStr{ - "free": totalSizeFree, - "used": totalSizeUsed, - "total": totalSize, - }, - "count": len(fss), - "total_files": totalFiles, + event := common.MapStr{ + "total_size": common.MapStr{ + "free": totalSizeFree, + "used": totalSizeUsed, + "total": totalSize, }, + "count": len(fss), + "total_files": totalFiles, + } + + //We don't get the `Files` field on Windows + if runtime.GOOS == "windows" { + event["total_files"] = totalFiles + } + + r.Event(mb.Event{ + MetricSetFields: event, }) return nil diff --git a/metricbeat/module/system/memory/_meta/data.json b/metricbeat/module/system/memory/_meta/data.json index 01231bede4c..06abee40a35 100644 --- a/metricbeat/module/system/memory/_meta/data.json +++ b/metricbeat/module/system/memory/_meta/data.json @@ -15,13 +15,13 @@ "system": { "memory": { "actual": { - "free": 14247317504, + "free": 8461623296, "used": { - "bytes": 1407057920, - "pct": 0.0899 + "bytes": 7159164928, + "pct": 0.4583 } }, - "free": 4859097088, + "free": 1299234816, "hugepages": { "default_size": 2097152, "free": 0, @@ -41,49 +41,49 @@ }, "page_stats": { "direct_efficiency": { - "pct": 0.9976 + "pct": 0.9242 }, "kswapd_efficiency": { - "pct": 0.6213 + "pct": 0.7518 }, "pgfree": { - "pages": 4382105954 + "pages": 15924304810 }, "pgscan_direct": { - "pages": 485820 + "pages": 1185751 }, "pgscan_kswapd": { - "pages": 77390925 + "pages": 50008148 }, "pgsteal_direct": { - "pages": 484631 + "pages": 1095884 }, "pgsteal_kswapd": { - "pages": 48081976 + "pages": 37594071 } }, "swap": { - "free": 7846490112, + "free": 7823421440, "in": { - "pages": 1111 + "pages": 2702 }, "out": { - "pages": 20255 + "pages": 23582 }, "readahead": { - "cached": 28, - "pages": 65 + "cached": 554, + "pages": 986 }, "total": 7897870336, "used": { - "bytes": 51380224, - "pct": 0.0065 + "bytes": 74448896, + "pct": 0.0094 } }, - "total": 15654375424, + "total": 15620788224, "used": { - "bytes": 10795278336, - "pct": 0.6896 + "bytes": 14321553408, + "pct": 0.9168 } } } diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index 90283f801be..27e76b85489 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -20,12 +20,16 @@ package memory import ( + "fmt" + + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" mem "github.com/elastic/beats/v7/libbeat/metric/system/memory" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" - - "github.com/pkg/errors" + linux "github.com/elastic/beats/v7/metricbeat/module/linux/memory" + "github.com/elastic/beats/v7/metricbeat/module/system" ) func init() { @@ -38,11 +42,18 @@ func init() { // MetricSet for fetching system memory metrics. type MetricSet struct { mb.BaseMetricSet + IsFleet bool } // New is a mb.MetricSetFactory that returns a memory.MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return &MetricSet{base}, nil + + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + + return &MetricSet{BaseMetricSet: base, IsFleet: systemModule.IsAgent}, nil } // Fetch fetches memory metrics from the OS. @@ -103,70 +114,18 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { "pages": vmstat.SwapRa, "cached": vmstat.SwapRaHit, } - pageStats := common.MapStr{ - "pgscan_kswapd": common.MapStr{ - "pages": vmstat.PgscanKswapd, - }, - "pgscan_direct": common.MapStr{ - "pages": vmstat.PgscanDirect, - }, - "pgfree": common.MapStr{ - "pages": vmstat.Pgfree, - }, - "pgsteal_kswapd": common.MapStr{ - "pages": vmstat.PgstealKswapd, - }, - "pgsteal_direct": common.MapStr{ - "pages": vmstat.PgstealDirect, - }, - } - // This is similar to the vmeff stat gathered by sar - // these ratios calculate thhe efficiency of page reclaim - if vmstat.PgscanDirect != 0 { - pageStats["direct_efficiency"] = common.MapStr{ - "pct": common.Round(float64(vmstat.PgstealDirect)/float64(vmstat.PgscanDirect), common.DefaultDecimalPlacesCount), - } - } + } - if vmstat.PgscanKswapd != 0 { - pageStats["kswapd_efficiency"] = common.MapStr{ - "pct": common.Round(float64(vmstat.PgstealKswapd)/float64(vmstat.PgscanKswapd), common.DefaultDecimalPlacesCount), - } + // for backwards compatibility, only report if we're not in fleet mode + if !m.IsFleet { + err := linux.FetchLinuxMemStats(memory) + if err != nil { + return errors.Wrap(err, "error getting page stats") } - - memory["page_stats"] = pageStats } memory["swap"] = swap - hugePagesStat, err := mem.GetHugeTLBPages() - if err != nil { - return errors.Wrap(err, "hugepages") - } - if hugePagesStat != nil { - mem.AddHugeTLBPagesPercentage(hugePagesStat) - thp := common.MapStr{ - "total": hugePagesStat.Total, - "used": common.MapStr{ - "bytes": hugePagesStat.TotalAllocatedSize, - "pct": hugePagesStat.UsedPercent, - }, - "free": hugePagesStat.Free, - "reserved": hugePagesStat.Reserved, - "surplus": hugePagesStat.Surplus, - "default_size": hugePagesStat.DefaultSize, - } - if vmstat != nil { - thp["swap"] = common.MapStr{ - "out": common.MapStr{ - "pages": vmstat.ThpSwpout, - "fallback": vmstat.ThpSwpoutFallback, - }, - } - } - memory["hugepages"] = thp - } - r.Event(mb.Event{ MetricSetFields: memory, }) diff --git a/metricbeat/module/system/process/_meta/data.json b/metricbeat/module/system/process/_meta/data.json index 78ece4aa114..84b74110683 100644 --- a/metricbeat/module/system/process/_meta/data.json +++ b/metricbeat/module/system/process/_meta/data.json @@ -1,29 +1,26 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.process", "duration": 115000, "module": "system" }, "metricset": { - "name": "process" + "name": "process", + "period": 10000 }, "process": { "args": [ - "/usr/bin/dockerd", - "-H", - "unix://" + "/usr/lib/systemd/systemd", + "--switched-root", + "--system", + "--deserialize", + "28" ], - "executable": "/usr/bin/dockerd-ce", - "name": "dockerd", - "pgid": 2080, - "pid": 2080, - "ppid": 1, - "working_directory": "/" + "name": "systemd", + "pgid": 1, + "pid": 1, + "ppid": 0 }, "service": { "type": "system" @@ -32,11 +29,11 @@ "process": { "cgroup": { "blkio": { - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "total": { - "bytes": 844576104448, - "ios": 54869430 + "bytes": 7453696, + "ios": 548 } }, "cpu": { @@ -49,8 +46,8 @@ }, "shares": 1024 }, - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "rt": { "period": { "us": 0 @@ -68,38 +65,38 @@ } }, "cpuacct": { - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "percpu": { - "1": 7058282754012, - "2": 7053634662537, - "3": 7069386293853, - "4": 7050055153087 + "1": 3930656993407, + "2": 4025787490535, + "3": 4064460082910, + "4": 3387847262532 }, "stats": { "system": { - "ns": 7202920000000 + "ns": 4996000000000 }, "user": { - "ns": 19573240000000 + "ns": 10329380000000 } }, "total": { - "ns": 28231358863489 + "ns": 15408751829384 } }, - "id": "docker.service", + "id": "init.scope", "memory": { - "id": "docker.service", + "id": "init.scope", "kmem": { "failures": 0, "limit": { "bytes": 9223372036854771712 }, "usage": { - "bytes": 21139456, + "bytes": 9404416, "max": { - "bytes": 480030720 + "bytes": 14987264 } } }, @@ -121,95 +118,88 @@ "bytes": 9223372036854771712 }, "usage": { - "bytes": 3337703424, + "bytes": 29437952, "max": { - "bytes": 5245300736 + "bytes": 70705152 } } }, "memsw": { "failures": 0, "limit": { - "bytes": 0 + "bytes": 9223372036854771712 }, "usage": { - "bytes": 0, + "bytes": 30392320, "max": { - "bytes": 0 + "bytes": 70705152 } } }, - "path": "/system.slice/docker.service", + "path": "/init.scope", "stats": { "active_anon": { - "bytes": 779677696 + "bytes": 3444736 }, "active_file": { - "bytes": 1753378816 + "bytes": 10563584 }, "cache": { - "bytes": 2127810560 + "bytes": 10752000 }, "hierarchical_memory_limit": { "bytes": 9223372036854771712 }, "hierarchical_memsw_limit": { - "bytes": 0 + "bytes": 9223372036854771712 }, "inactive_anon": { - "bytes": 409075712 + "bytes": 6197248 }, "inactive_file": { - "bytes": 374431744 + "bytes": 327680 }, - "major_page_faults": 53164, + "major_page_faults": 198, "mapped_file": { - "bytes": 7491584 + "bytes": 9867264 }, - "page_faults": 21923702, - "pages_in": 57261049, - "pages_out": 56521859, + "page_faults": 3626304, + "pages_in": 1095732, + "pages_out": 1090806, "rss": { - "bytes": 1188753408 + "bytes": 9592832 }, "rss_huge": { - "bytes": 56623104 + "bytes": 0 }, "swap": { - "bytes": 0 + "bytes": 675840 }, "unevictable": { "bytes": 0 } } }, - "path": "/system.slice/docker.service" + "path": "/init.scope" }, - "cmdline": "/usr/bin/dockerd -H unix://", + "cmdline": "/usr/lib/systemd/systemd --switched-root --system --deserialize 28", "cpu": { - "start_time": "2019-01-08T17:06:39.000Z", + "start_time": "2020-08-27T01:05:16.000Z", "total": { "norm": { - "pct": 0.0049 + "pct": 0.0056 }, - "pct": 0.0195, - "value": 27827810 + "pct": 0.0222, + "value": 15389060 } }, - "fd": { - "limit": { - "hard": 1048576, - "soft": 1048576 - }, - "open": 68 - }, "memory": { "rss": { - "bytes": 1187336192, - "pct": 0.0716 + "bytes": 12853248, + "pct": 0.0008 }, - "share": 8253440, - "size": 4438523904 + "share": 7118848, + "size": 176881664 }, "state": "sleeping" } @@ -217,4 +207,4 @@ "user": { "name": "root" } -} +} \ No newline at end of file diff --git a/metricbeat/module/system/process/_meta/fields.yml b/metricbeat/module/system/process/_meta/fields.yml index 51ab8b81e09..fe941031551 100644 --- a/metricbeat/module/system/process/_meta/fields.yml +++ b/metricbeat/module/system/process/_meta/fields.yml @@ -96,17 +96,27 @@ type: long format: bytes description: > - The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. + The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. Not available on Windows. - name: rss.pct type: scaled_float format: percent description: > - The percentage of memory the process occupied in main memory (RAM). + The percentage of memory the process occupied in main memory (RAM). Not available on Windows. + - name: wss.bytes + type: long + format: bytes + description: > + The Working Set Size. The amount of memory the process occupied in main memory (RAM). Windows only. + - name: wss.pct + type: scaled_float + format: percent + description: > + The percentage of memory the process occupied in main memory (RAM). Windows only. - name: share type: long format: bytes description: > - The shared memory the process uses. + The shared memory the process uses. Not available on Windows. - name: fd type: group description: > diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go index 9c754177372..804c62d06d6 100644 --- a/metricbeat/module/system/process/process.go +++ b/metricbeat/module/system/process/process.go @@ -46,9 +46,10 @@ func init() { // MetricSet that fetches process metrics. type MetricSet struct { mb.BaseMetricSet - stats *process.Stats - cgroup *cgroup.Reader - perCPU bool + stats *process.Stats + cgroup *cgroup.Reader + perCPU bool + IsAgent bool } // New creates and returns a new MetricSet. @@ -58,6 +59,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + m := &MetricSet{ BaseMetricSet: base, stats: &process.Stats{ @@ -67,7 +73,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { CacheCmdLine: config.CacheCmdLine, IncludeTop: config.IncludeTop, }, - perCPU: config.IncludePerCPU, + perCPU: config.IncludePerCPU, + IsAgent: systemModule.IsAgent, } err := m.stats.Init() if err != nil { @@ -75,11 +82,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } if runtime.GOOS == "linux" { - systemModule, ok := base.Module().(*system.Module) - if !ok { - return nil, fmt.Errorf("unexpected module type") - } - if config.Cgroups == nil || *config.Cgroups { debugf("process cgroup data collection is enabled, using hostfs='%v'", systemModule.HostFS) m.cgroup, err = cgroup.NewReader(systemModule.HostFS, true) @@ -148,6 +150,22 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { rootFields.Put("process.args", args) } + // This is a temporary fix until we make these changes global across libbeat + // This logic should happen in libbeat getProcessEvent() + + // There's some more Windows memory quirks we need to deal with. + // "rss" is a linux concept, but "wss" is a direct match on Windows. + // "share" is also unavailable on Windows. + + if m.IsAgent { + if runtime.GOOS == "windows" { + proc.Delete("memory.share") + if setSize := getAndRemove(proc, "memory.rss"); setSize != nil { + proc.Put("memory.wss", setSize) + } + } + } + e := mb.Event{ RootFields: rootFields, MetricSetFields: proc, diff --git a/metricbeat/module/system/process_summary/_meta/data.json b/metricbeat/module/system/process_summary/_meta/data.json index 363e5f77a0d..159defcb769 100644 --- a/metricbeat/module/system/process_summary/_meta/data.json +++ b/metricbeat/module/system/process_summary/_meta/data.json @@ -1,16 +1,13 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.process.summary", "duration": 115000, "module": "system" }, "metricset": { - "name": "process_summary" + "name": "process_summary", + "period": 10000 }, "service": { "type": "system" @@ -19,13 +16,13 @@ "process": { "summary": { "dead": 0, - "idle": 112, - "running": 0, - "sleeping": 288, + "idle": 63, + "running": 3, + "sleeping": 117, "stopped": 0, - "total": 401, - "unknown": 0, - "zombie": 1 + "total": 185, + "unknown": 2, + "zombie": 0 } } } diff --git a/metricbeat/module/system/process_summary/process_summary.go b/metricbeat/module/system/process_summary/process_summary.go index 071a7b2a660..5edb31f9182 100644 --- a/metricbeat/module/system/process_summary/process_summary.go +++ b/metricbeat/module/system/process_summary/process_summary.go @@ -20,6 +20,8 @@ package process_summary import ( + "runtime" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -79,7 +81,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { state := sigar.ProcState{} err = state.Get(pid) if err != nil { - summary.unknown += 1 + summary.unknown++ continue } @@ -104,15 +106,25 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } } - event := common.MapStr{ - "total": len(pids), - "sleeping": summary.sleeping, - "running": summary.running, - "idle": summary.idle, - "stopped": summary.stopped, - "zombie": summary.zombie, - "unknown": summary.unknown, - "dead": summary.dead, + event := common.MapStr{} + if runtime.GOOS == "windows" { + event = common.MapStr{ + "total": len(pids), + "sleeping": summary.sleeping, + "running": summary.running, + "unknown": summary.unknown, + } + } else { + event = common.MapStr{ + "total": len(pids), + "sleeping": summary.sleeping, + "running": summary.running, + "idle": summary.idle, + "stopped": summary.stopped, + "zombie": summary.zombie, + "unknown": summary.unknown, + "dead": summary.dead, + } } r.Event(mb.Event{ diff --git a/metricbeat/module/system/process_summary/process_summary_test.go b/metricbeat/module/system/process_summary/process_summary_test.go index 55afce8f190..c95e361224c 100644 --- a/metricbeat/module/system/process_summary/process_summary_test.go +++ b/metricbeat/module/system/process_summary/process_summary_test.go @@ -20,6 +20,7 @@ package process_summary import ( + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -53,18 +54,26 @@ func TestFetch(t *testing.T) { event, ok := summary.(common.MapStr) require.True(t, ok) - assert.Contains(t, event, "total") - assert.Contains(t, event, "sleeping") - assert.Contains(t, event, "running") - assert.Contains(t, event, "idle") - assert.Contains(t, event, "stopped") - assert.Contains(t, event, "zombie") - assert.Contains(t, event, "unknown") + if runtime.GOOS == "windows" { + assert.Contains(t, event, "total") + assert.Contains(t, event, "sleeping") + assert.Contains(t, event, "running") + assert.Contains(t, event, "unknown") + total := event["sleeping"].(int) + event["running"].(int) + event["unknown"].(int) + assert.Equal(t, event["total"].(int), total) + } else { + assert.Contains(t, event, "total") + assert.Contains(t, event, "sleeping") + assert.Contains(t, event, "running") + assert.Contains(t, event, "idle") + assert.Contains(t, event, "stopped") + assert.Contains(t, event, "zombie") + assert.Contains(t, event, "unknown") + total := event["sleeping"].(int) + event["running"].(int) + event["idle"].(int) + + event["stopped"].(int) + event["zombie"].(int) + event["unknown"].(int) - total := event["sleeping"].(int) + event["running"].(int) + event["idle"].(int) + - event["stopped"].(int) + event["zombie"].(int) + event["unknown"].(int) - - assert.Equal(t, event["total"].(int), total) + assert.Equal(t, event["total"].(int), total) + } } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/system.go b/metricbeat/module/system/system.go index edb9f55bf6b..f1f060a90bc 100644 --- a/metricbeat/module/system/system.go +++ b/metricbeat/module/system/system.go @@ -21,10 +21,12 @@ import ( "flag" "sync" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) var ( + // HostFS is an alternate mountpoint for the filesytem root, for when metricbeat is running inside a container. HostFS = flag.String("system.hostfs", "", "mountpoint of the host's filesystem for use in monitoring a host from within a container") ) @@ -37,16 +39,48 @@ func init() { } } +// Module represents the system module type Module struct { mb.BaseModule - HostFS string // Mountpoint of the host's filesystem for use in monitoring inside a container. + HostFS string // Mountpoint of the host's filesystem for use in monitoring inside a container. + IsAgent bool // Looks to see if metricbeat is running under agent. Useful if we have breaking changes in one but not the other. } +// NewModule instatiates the system module func NewModule(base mb.BaseModule) (mb.Module, error) { // This only needs to be configured once for all system modules. once.Do(func() { initModule() }) - return &Module{BaseModule: base, HostFS: *HostFS}, nil + return &Module{BaseModule: base, HostFS: *HostFS, IsAgent: checkMgmtFlags()}, nil +} + +// checkMgmtFlags checks to see if metricbeat is running under Agent +// The management setting is stored in the main Beat runtime object, but we can't see that from a module +// So instead we check the CLI flags, since Agent starts metricbeat with "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true" +func checkMgmtFlags() bool { + type management struct { + Mode string `config:"management.mode"` + Enabled bool `config:"management.enabled"` + } + var managementSettings management + + cfgFlag := flag.Lookup("E") + if cfgFlag == nil { + return false + } + + CfgObject, _ := cfgFlag.Value.(*common.SettingsFlag) + cliCfg := CfgObject.Config() + + err := cliCfg.Unpack(&managementSettings) + if err != nil { + return false + } + + if managementSettings.Enabled == true && managementSettings.Mode == "x-pack-fleet" { + return true + } + return false } diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index f689b99fb4c..2e1b9e579d9 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -7,13 +7,23 @@ import unittest -SYSTEM_CPU_FIELDS = ["cores", "idle.pct", "iowait.pct", "irq.pct", "nice.pct", - "softirq.pct", "steal.pct", "system.pct", "user.pct", "total.pct"] +SYSTEM_CPU_FIELDS_LINUX = ["cores", "idle.pct", "iowait.pct", "irq.pct", "nice.pct", + "softirq.pct", "steal.pct", "system.pct", "user.pct", "total.pct"] -SYSTEM_CPU_FIELDS_ALL = ["cores", "idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks", - "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", - "idle.norm.pct", "iowait.norm.pct", "irq.norm.pct", "nice.norm.pct", "softirq.norm.pct", - "steal.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] +SYSTEM_CPU_FIELDS_WINDOWS = ["cores", "idle.pct", "system.pct", "user.pct", "total.pct"] + +SYSTEM_CPU_FIELDS_DARWIN = ["cores", "idle.pct", "system.pct", "user.pct", "total.pct", "nice.pct"] + +SYSTEM_CPU_FIELDS_LINUX_ALL = ["cores", "idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks", + "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "iowait.norm.pct", "irq.norm.pct", "nice.norm.pct", "softirq.norm.pct", + "steal.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] + +SYSTEM_CPU_FIELDS_WINDOWS_ALL = ["cores", "idle.pct", "idle.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] + +SYSTEM_CPU_FIELDS_DARWIN_ALL = ["cores", "idle.pct", "idle.ticks", "nice.pct", "nice.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "nice.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] SYSTEM_LOAD_FIELDS = ["cores", "1", "5", "15", "norm.1", "norm.5", "norm.15"] @@ -37,6 +47,10 @@ "free_files", "mount_point", "total", "used.bytes", "used.pct"] +SYSTEM_FILESYSTEM_FIELDS_WINDOWS = ["available", "device_name", "type", "free", + "mount_point", "total", "used.bytes", + "used.pct"] + SYSTEM_FSSTAT_FIELDS = ["count", "total_files", "total_size"] SYSTEM_MEMORY_FIELDS = ["swap", "actual.free", "free", "total", "used.bytes", "used.pct", "actual.used.bytes", @@ -82,7 +96,12 @@ def test_cpu(self): if "system" in evt: cpu = evt["system"]["cpu"] - self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) + if sys.platform.startswith("linux"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_LINUX), cpu.keys()) + elif sys.platform.startswith("darwin"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_DARWIN), cpu.keys()) + elif sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_WINDOWS), cpu.keys()) else: host_cpu = evt["host"]["cpu"] self.assertCountEqual(self.de_dot(SYSTEM_CPU_HOST_FIELDS), host_cpu.keys()) @@ -111,7 +130,12 @@ def test_cpu_ticks_option(self): for evt in output: self.assert_fields_are_documented(evt) cpuStats = evt["system"]["cpu"] - self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_ALL), cpuStats.keys()) + if sys.platform.startswith("linux"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_LINUX_ALL), cpuStats.keys()) + elif sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_WINDOWS_ALL), cpuStats.keys()) + elif sys.platform.startswith("darwin"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_DARWIN_ALL), cpuStats.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_core(self): @@ -261,7 +285,10 @@ def test_filesystem(self): for evt in output: self.assert_fields_are_documented(evt) filesystem = evt["system"]["filesystem"] - self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) + if sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS_WINDOWS), filesystem.keys()) + else: + self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_fsstat(self): @@ -378,13 +405,17 @@ def test_process_summary(self): assert isinstance(summary["total"], int) assert isinstance(summary["sleeping"], int) assert isinstance(summary["running"], int) - assert isinstance(summary["idle"], int) - assert isinstance(summary["stopped"], int) - assert isinstance(summary["zombie"], int) assert isinstance(summary["unknown"], int) - assert summary["total"] == summary["sleeping"] + summary["running"] + \ - summary["idle"] + summary["stopped"] + summary["zombie"] + summary["unknown"] + if not sys.platform.startswith("win"): + assert isinstance(summary["idle"], int) + assert isinstance(summary["stopped"], int) + assert isinstance(summary["zombie"], int) + assert summary["total"] == summary["sleeping"] + summary["running"] + \ + summary["idle"] + summary["stopped"] + summary["zombie"] + summary["unknown"] + + if sys.platform.startswith("windows"): + assert summary["total"] == summary["sleeping"] + summary["running"] + summary["unknown"] @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process(self): diff --git a/metricbeat/modules.d/linux.yml.disabled b/metricbeat/modules.d/linux.yml.disabled index a01989aa0c9..22e675cafff 100644 --- a/metricbeat/modules.d/linux.yml.disabled +++ b/metricbeat/modules.d/linux.yml.disabled @@ -5,8 +5,10 @@ period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/tests/system/test_processors.py b/metricbeat/tests/system/test_processors.py index 2f7d131d119..a680c049d67 100644 --- a/metricbeat/tests/system/test_processors.py +++ b/metricbeat/tests/system/test_processors.py @@ -12,13 +12,13 @@ def test_drop_fields(self): self.render_config_template( modules=[{ "name": "system", - "metricsets": ["cpu"], + "metricsets": ["network"], "period": "1s" }], processors=[{ "drop_fields": { "when": "range.system.cpu.system.pct.lt: 0.1", - "fields": ["system.cpu.load"], + "fields": ["system.network.in"], }, }] ) @@ -27,7 +27,7 @@ def test_drop_fields(self): proc.check_kill_and_wait() output = self.read_output_json() - self.assertEqual(len(output), 1) + self.assertGreater(len(output), 1) evt = output[0] self.assert_fields_are_documented(evt) @@ -37,12 +37,9 @@ def test_drop_fields(self): 'agent', '@timestamp', 'system', 'metricset.module', 'metricset.rtt', 'metricset.name', 'host', 'service', 'ecs', 'event' ]), evt.keys()) - cpu = evt["system"]["cpu"] - print(list(cpu.keys())) - self.assertCountEqual(self.de_dot([ - "system", "cores", "user", "softirq", "iowait", - "idle", "irq", "steal", "nice", "total" - ]), cpu.keys()) + network = evt["system"]["network"] + print(list(network.keys())) + self.assertCountEqual(self.de_dot(["name", "out", "in"]), network.keys()) def test_dropfields_with_condition(self): """ diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 1d252b36b31..7ecdbbfca8f 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -45,3 +45,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test packetbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/packetbeat/_meta/config/beat.docker.yml.tmpl b/packetbeat/_meta/config/beat.docker.yml.tmpl index f4f0db1f7e6..f9b573edfeb 100644 --- a/packetbeat/_meta/config/beat.docker.yml.tmpl +++ b/packetbeat/_meta/config/beat.docker.yml.tmpl @@ -36,3 +36,6 @@ packetbeat.protocols.cassandra: packetbeat.protocols.tls: ports: [443, 993, 995, 5223, 8443, 8883, 9243] + +packetbeat.protocols.sip: + ports: [5060] diff --git a/packetbeat/_meta/config/beat.reference.yml.tmpl b/packetbeat/_meta/config/beat.reference.yml.tmpl index 1a3aab315d7..722c47102dc 100644 --- a/packetbeat/_meta/config/beat.reference.yml.tmpl +++ b/packetbeat/_meta/config/beat.reference.yml.tmpl @@ -531,6 +531,19 @@ packetbeat.protocols: # Set to true to publish fields with null values in events. #keep_null: false +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + + # Parse the authorization headers + parse_authorization: true + + # Parse body contents (only when body is SDP) + parse_body: true + + # Preserve original contents in event.original + keep_original: true + {{header "Monitored processes"}} # Packetbeat can enrich events with information about the process associated diff --git a/packetbeat/_meta/config/beat.yml.tmpl b/packetbeat/_meta/config/beat.yml.tmpl index fb221cba3c9..c5f9cfc0c23 100644 --- a/packetbeat/_meta/config/beat.yml.tmpl +++ b/packetbeat/_meta/config/beat.yml.tmpl @@ -101,6 +101,10 @@ packetbeat.protocols: - 8883 # Secure MQTT - 9243 # Elasticsearch +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + {{header "Elasticsearch template setting"}} setup.template.settings: diff --git a/packetbeat/_meta/sample_outputs/sip.json b/packetbeat/_meta/sample_outputs/sip.json new file mode 100644 index 00000000000..4a57d85908f --- /dev/null +++ b/packetbeat/_meta/sample_outputs/sip.json @@ -0,0 +1,77 @@ +{ + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "192.168.1.2", + "client.port": 5060, + "destination.ip": "212.242.33.35", + "destination.port": 5060, + "event.action": "sip_register", + "event.category": [ + "network", + "protocol", + "authentication" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "REGISTER sip:sip.cybercity.dk SIP/2.0\r\nVia: SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport\r\nFrom: ;tag=6bac55c\r\nTo: \r\nCall-ID: 578222729-4665d775@578222732-4665d772\r\nContact: ;expires=1200;q=0.500\r\nExpires: 1200\r\nCSeq: 75 REGISTER\r\nContent-Length: 0\r\nAuthorization: Digest username=\"voi18062\",realm=\"sip.cybercity.dk\",uri=\"sip:192.168.1.2\",nonce=\"1701b22972b90f440c3e4eb250842bb\",opaque=\"1701a1351f70795\",nc=\"00000001\",response=\"79a0543188495d288c9ebbe0c881abdc\"\r\nMax-Forwards: 70\r\nUser-Agent: Nero SIPPS IP Phone Version 2.0.51.16\r\n\r\n", + "event.sequence": 75, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:dOa61R2NaaJsJlcFAiMIiyXX+Kk=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "sip.cybercity.dk" + ], + "related.ip": [ + "192.168.1.2", + "212.242.33.35" + ], + "related.user": [ + "voi18062" + ], + "server.ip": "212.242.33.35", + "server.port": 5060, + "sip.auth.realm": "sip.cybercity.dk", + "sip.auth.scheme": "Digest", + "sip.auth.uri.host": "192.168.1.2", + "sip.auth.uri.original": "sip:192.168.1.2", + "sip.auth.uri.scheme": "sip", + "sip.call_id": "578222729-4665d775@578222732-4665d772", + "sip.contact.uri.host": "sip.cybercity.dk", + "sip.contact.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "voi18062", + "sip.cseq.code": 75, + "sip.cseq.method": "REGISTER", + "sip.from.tag": "6bac55c", + "sip.from.uri.host": "sip.cybercity.dk", + "sip.from.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "voi18062", + "sip.max_forwards": 70, + "sip.method": "REGISTER", + "sip.to.uri.host": "sip.cybercity.dk", + "sip.to.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "voi18062", + "sip.type": "request", + "sip.uri.host": "sip.cybercity.dk", + "sip.uri.original": "sip:sip.cybercity.dk", + "sip.uri.scheme": "sip", + "sip.user_agent.original": "Nero SIPPS IP Phone Version 2.0.51.16", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport" + ], + "source.ip": "192.168.1.2", + "source.port": 5060, + "status": "OK", + "type": "sip", + "user.name": "voi18062" +} diff --git a/packetbeat/cmd/root.go b/packetbeat/cmd/root.go index 82ab41da374..0a22e98a1a0 100644 --- a/packetbeat/cmd/root.go +++ b/packetbeat/cmd/root.go @@ -37,7 +37,7 @@ const ( Name = "packetbeat" // ecsVersion specifies the version of ECS that Packetbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // withECSVersion is a modifier that adds ecs.version to events. diff --git a/packetbeat/docs/fields.asciidoc b/packetbeat/docs/fields.asciidoc index 2c73f3dd277..22d2b406bf9 100644 --- a/packetbeat/docs/fields.asciidoc +++ b/packetbeat/docs/fields.asciidoc @@ -35,6 +35,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -2127,8 +2128,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + @@ -11680,6 +11688,667 @@ The return value of the Redis command in a human readable format. If the Redis command has resulted in an error, this field contains the error message returned by the Redis server. +-- + +[[exported-fields-sip]] +== SIP fields + +SIP-specific event fields. + + +[float] +=== sip + +Information about SIP traffic. + + +*`sip.timestamp`*:: ++ +-- +Timestamp with nano second precision. + +type: date_nanos + +-- + +*`sip.code`*:: ++ +-- +Response status code. + +type: keyword + +-- + +*`sip.method`*:: ++ +-- +Request method. + +type: keyword + +-- + +*`sip.status`*:: ++ +-- +Response status phrase. + +type: keyword + +-- + +*`sip.type`*:: ++ +-- +Either request or response. + +type: keyword + +-- + +*`sip.version`*:: ++ +-- +SIP protocol version. + +type: keyword + +-- + +*`sip.uri.original`*:: ++ +-- +The original URI. + +type: keyword + +-- + +*`sip.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.uri.scheme`*:: ++ +-- +The URI scheme. + +type: keyword + +-- + +*`sip.uri.username`*:: ++ +-- +The URI user name. + +type: keyword + +-- + +*`sip.uri.host`*:: ++ +-- +The URI host. + +type: keyword + +-- + +*`sip.uri.port`*:: ++ +-- +The URI port. + +type: keyword + +-- + +*`sip.accept`*:: ++ +-- +Accept header value. + +type: keyword + +-- + +*`sip.allow`*:: ++ +-- +Allowed methods. + +type: keyword + +-- + +*`sip.call_id`*:: ++ +-- +Call ID. + +type: keyword + +-- + +*`sip.content_length`*:: ++ +-- +type: long + +-- + +*`sip.content_type`*:: ++ +-- +type: keyword + +-- + +*`sip.max_forwards`*:: ++ +-- +type: long + +-- + +*`sip.supported`*:: ++ +-- +Supported methods. + +type: keyword + +-- + +*`sip.user_agent.original`*:: ++ +-- +type: keyword + +-- + +*`sip.user_agent.original.text`*:: ++ +-- +type: text + +-- + +*`sip.private.uri.original`*:: ++ +-- +Private original URI. + +type: keyword + +-- + +*`sip.private.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.private.uri.scheme`*:: ++ +-- +Private URI scheme. + +type: keyword + +-- + +*`sip.private.uri.username`*:: ++ +-- +Private URI user name. + +type: keyword + +-- + +*`sip.private.uri.host`*:: ++ +-- +Private URI host. + +type: keyword + +-- + +*`sip.private.uri.port`*:: ++ +-- +Private URI port. + +type: keyword + +-- + +*`sip.cseq.code`*:: ++ +-- +Sequence code. + +type: keyword + +-- + +*`sip.cseq.method`*:: ++ +-- +Sequence method. + +type: keyword + +-- + +*`sip.via.original`*:: ++ +-- +The original Via value. + +type: keyword + +-- + +*`sip.via.original.text`*:: ++ +-- +type: text + +-- + +*`sip.to.display_info`*:: ++ +-- +To display info + +type: keyword + +-- + +*`sip.to.uri.original`*:: ++ +-- +To original URI + +type: keyword + +-- + +*`sip.to.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.to.uri.scheme`*:: ++ +-- +To URI scheme + +type: keyword + +-- + +*`sip.to.uri.username`*:: ++ +-- +To URI user name + +type: keyword + +-- + +*`sip.to.uri.host`*:: ++ +-- +To URI host + +type: keyword + +-- + +*`sip.to.uri.port`*:: ++ +-- +To URI port + +type: keyword + +-- + +*`sip.to.tag`*:: ++ +-- +To tag + +type: keyword + +-- + +*`sip.from.display_info`*:: ++ +-- +From display info + +type: keyword + +-- + +*`sip.from.uri.original`*:: ++ +-- +From original URI + +type: keyword + +-- + +*`sip.from.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.from.uri.scheme`*:: ++ +-- +From URI scheme + +type: keyword + +-- + +*`sip.from.uri.username`*:: ++ +-- +From URI user name + +type: keyword + +-- + +*`sip.from.uri.host`*:: ++ +-- +From URI host + +type: keyword + +-- + +*`sip.from.uri.port`*:: ++ +-- +From URI port + +type: keyword + +-- + +*`sip.from.tag`*:: ++ +-- +From tag + +type: keyword + +-- + +*`sip.contact.display_info`*:: ++ +-- +Contact display info + +type: keyword + +-- + +*`sip.contact.uri.original`*:: ++ +-- +Contact original URI + +type: keyword + +-- + +*`sip.contact.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.contact.uri.scheme`*:: ++ +-- +Contat URI scheme + +type: keyword + +-- + +*`sip.contact.uri.username`*:: ++ +-- +Contact URI user name + +type: keyword + +-- + +*`sip.contact.uri.host`*:: ++ +-- +Contact URI host + +type: keyword + +-- + +*`sip.contact.uri.port`*:: ++ +-- +Contact URI port + +type: keyword + +-- + +*`sip.contact.transport`*:: ++ +-- +Contact transport + +type: keyword + +-- + +*`sip.contact.line`*:: ++ +-- +Contact line + +type: keyword + +-- + +*`sip.contact.expires`*:: ++ +-- +Contact expires + +type: keyword + +-- + +*`sip.contact.q`*:: ++ +-- +Contact Q + +type: keyword + +-- + +*`sip.auth.scheme`*:: ++ +-- +Auth scheme + +type: keyword + +-- + +*`sip.auth.realm`*:: ++ +-- +Auth realm + +type: keyword + +-- + +*`sip.auth.uri.original`*:: ++ +-- +Auth original URI + +type: keyword + +-- + +*`sip.auth.uri.original.text`*:: ++ +-- +type: text + +-- + +*`sip.auth.uri.scheme`*:: ++ +-- +Auth URI scheme + +type: keyword + +-- + +*`sip.auth.uri.host`*:: ++ +-- +Auth URI host + +type: keyword + +-- + +*`sip.auth.uri.port`*:: ++ +-- +Auth URI port + +type: keyword + +-- + +*`sip.sdp.version`*:: ++ +-- +SDP version + +type: keyword + +-- + +*`sip.sdp.owner.username`*:: ++ +-- +SDP owner user name + +type: keyword + +-- + +*`sip.sdp.owner.session_id`*:: ++ +-- +SDP owner session ID + +type: keyword + +-- + +*`sip.sdp.owner.version`*:: ++ +-- +SDP owner version + +type: keyword + +-- + +*`sip.sdp.owner.ip`*:: ++ +-- +SDP owner IP + +type: ip + +-- + +*`sip.sdp.session.name`*:: ++ +-- +SDP session name + +type: keyword + +-- + +*`sip.sdp.connection.info`*:: ++ +-- +SDP connection info + +type: keyword + +-- + +*`sip.sdp.connection.address`*:: ++ +-- +SDP connection address + +type: keyword + +-- + +*`sip.sdp.body.original`*:: ++ +-- +SDP original body + +type: keyword + +-- + +*`sip.sdp.body.original.text`*:: ++ +-- +type: text + -- [[exported-fields-thrift]] diff --git a/packetbeat/docs/shared-protocol-list.asciidoc b/packetbeat/docs/shared-protocol-list.asciidoc index 3e18fc35eb8..fdac127050a 100644 --- a/packetbeat/docs/shared-protocol-list.asciidoc +++ b/packetbeat/docs/shared-protocol-list.asciidoc @@ -18,3 +18,4 @@ - Memcache - NFS - TLS + - SIP/SDP (beta) diff --git a/packetbeat/include/list.go b/packetbeat/include/list.go index 0dc1f0bd053..748d525eb2f 100644 --- a/packetbeat/include/list.go +++ b/packetbeat/include/list.go @@ -33,6 +33,7 @@ import ( _ "github.com/elastic/beats/v7/packetbeat/protos/nfs" _ "github.com/elastic/beats/v7/packetbeat/protos/pgsql" _ "github.com/elastic/beats/v7/packetbeat/protos/redis" + _ "github.com/elastic/beats/v7/packetbeat/protos/sip" _ "github.com/elastic/beats/v7/packetbeat/protos/thrift" _ "github.com/elastic/beats/v7/packetbeat/protos/tls" ) diff --git a/packetbeat/packetbeat.docker.yml b/packetbeat/packetbeat.docker.yml index edffce72694..4cf9016a926 100644 --- a/packetbeat/packetbeat.docker.yml +++ b/packetbeat/packetbeat.docker.yml @@ -37,6 +37,9 @@ packetbeat.protocols.cassandra: packetbeat.protocols.tls: ports: [443, 993, 995, 5223, 8443, 8883, 9243] +packetbeat.protocols.sip: + ports: [5060] + processors: - add_cloud_metadata: ~ - add_docker_metadata: ~ diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 0dc551698e9..6b936240bbb 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -531,6 +531,19 @@ packetbeat.protocols: # Set to true to publish fields with null values in events. #keep_null: false +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + + # Parse the authorization headers + parse_authorization: true + + # Parse body contents (only when body is SDP) + parse_body: true + + # Preserve original contents in event.original + keep_original: true + # ============================ Monitored processes ============================= # Packetbeat can enrich events with information about the process associated diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 53f87d73003..31c229b1ef7 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -101,6 +101,10 @@ packetbeat.protocols: - 8883 # Secure MQTT - 9243 # Elasticsearch +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + # ======================= Elasticsearch template setting ======================= setup.template.settings: diff --git a/packetbeat/pb/ecs.go b/packetbeat/pb/ecs.go index b7722c2c22a..4594f7cd8c4 100644 --- a/packetbeat/pb/ecs.go +++ b/packetbeat/pb/ecs.go @@ -54,7 +54,11 @@ type ecsRelated struct { User []string `ecs:"user"` // overridden because this needs to be an array Hash []string `ecs:"hash"` + // overridden because this needs to be an array + Hosts []string `ecs:"hosts"` // for de-dup - ipSet map[string]struct{} + ipSet map[string]struct{} + userSet map[string]struct{} + hostSet map[string]struct{} } diff --git a/packetbeat/pb/event.go b/packetbeat/pb/event.go index f0287665c0d..cd652756f3e 100644 --- a/packetbeat/pb/event.go +++ b/packetbeat/pb/event.go @@ -147,10 +147,15 @@ func (f *Fields) SetDestination(endpoint *common.Endpoint) { func (f *Fields) AddIP(ip ...string) { if f.Related == nil { f.Related = &ecsRelated{ - ipSet: make(map[string]struct{}), + ipSet: make(map[string]struct{}), + userSet: make(map[string]struct{}), + hostSet: make(map[string]struct{}), } } for _, ipAddress := range ip { + if ipAddress == "" { + continue + } if _, ok := f.Related.ipSet[ipAddress]; !ok { f.Related.ipSet[ipAddress] = struct{}{} f.Related.IP = append(f.Related.IP, ipAddress) @@ -158,6 +163,46 @@ func (f *Fields) AddIP(ip ...string) { } } +// AddUser adds the given user names to the related ECS User field +func (f *Fields) AddUser(u ...string) { + if f.Related == nil { + f.Related = &ecsRelated{ + ipSet: make(map[string]struct{}), + userSet: make(map[string]struct{}), + hostSet: make(map[string]struct{}), + } + } + for _, user := range u { + if user == "" { + continue + } + if _, ok := f.Related.userSet[user]; !ok { + f.Related.userSet[user] = struct{}{} + f.Related.User = append(f.Related.User, user) + } + } +} + +// AddHost adds the given hosts to the related ECS Hosts field +func (f *Fields) AddHost(h ...string) { + if f.Related == nil { + f.Related = &ecsRelated{ + ipSet: make(map[string]struct{}), + userSet: make(map[string]struct{}), + hostSet: make(map[string]struct{}), + } + } + for _, host := range h { + if host == "" { + continue + } + if _, ok := f.Related.hostSet[host]; !ok { + f.Related.hostSet[host] = struct{}{} + f.Related.Hosts = append(f.Related.Hosts, host) + } + } +} + func makeProcess(p *common.Process) *ecs.Process { return &ecs.Process{ Name: p.Name, diff --git a/packetbeat/protos/sip/README.md b/packetbeat/protos/sip/README.md new file mode 100644 index 00000000000..5e5962675a1 --- /dev/null +++ b/packetbeat/protos/sip/README.md @@ -0,0 +1,123 @@ +# SIP (Session Initiation Protocol) for packetbeat + +The SIP (Session Initiation Protocol) is a communications protocol for signaling and controlling multimedia communication sessions. SIP is used by many VoIP applications, not only for enterprise uses but also telecom carriers. + +SIP is a text-based protocol like HTTP. But SIP has various unique features like : +- SIP is server-client model, but its role may change in a per call basis. +- SIP is request-response model, but server may (usually) reply with many responses to a single request. +- There are many requests and responses in one call. +- It is not known when the call will end. + +## Implementation + +### Published for each SIP message (request or response) + +- SIP is not a one to one message with request and response. Also order to each message is not determined (a response may be sent after previous response). +- Therefore the SIP responses and requests are published when packetbeat receives them immediately. +- If you need all SIP messages in throughout of SIP dialog, you need to retrieve from Elasticsearch using the SIP Call ID field etc. + +### Notes +* ``transport=tcp`` is not supported yet. +* ``content-encoding`` is not supported yet. +* Default timestamp field(@timestamp) precision is not sufficient(the sip response is often send immediately when request received eg. 100 Trying). You can sort to keep the message correct order using the ``sip.timestamp``(`date_nanos`) field. +* Body parsing is partially supported for ``application/sdp`` content type only. + +## Configuration + +```yaml +- type: sip + # Configure the ports where to listen for SIP traffic. You can disable the SIP protocol by commenting out the list of ports. + ports: [5060] + + # Parse the authorization headers + parse_authorization: true + + # Parse body contents (only when body is SDP) + parse_body: true + + # Preserve original contents in event.original + keep_original: true +``` + +### Sample Full JSON Output + +```json +{ + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "192.168.1.2", + "client.port": 5060, + "destination.ip": "212.242.33.35", + "destination.port": 5060, + "event.action": "sip_register", + "event.category": [ + "network", + "protocol", + "authentication" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "REGISTER sip:sip.cybercity.dk SIP/2.0\r\nVia: SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport\r\nFrom: ;tag=6bac55c\r\nTo: \r\nCall-ID: 578222729-4665d775@578222732-4665d772\r\nContact: ;expires=1200;q=0.500\r\nExpires: 1200\r\nCSeq: 75 REGISTER\r\nContent-Length: 0\r\nAuthorization: Digest username=\"voi18062\",realm=\"sip.cybercity.dk\",uri=\"sip:192.168.1.2\",nonce=\"1701b22972b90f440c3e4eb250842bb\",opaque=\"1701a1351f70795\",nc=\"00000001\",response=\"79a0543188495d288c9ebbe0c881abdc\"\r\nMax-Forwards: 70\r\nUser-Agent: Nero SIPPS IP Phone Version 2.0.51.16\r\n\r\n", + "event.sequence": 75, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:dOa61R2NaaJsJlcFAiMIiyXX+Kk=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "sip.cybercity.dk" + ], + "related.ip": [ + "192.168.1.2", + "212.242.33.35" + ], + "related.user": [ + "voi18062" + ], + "server.ip": "212.242.33.35", + "server.port": 5060, + "sip.auth.realm": "sip.cybercity.dk", + "sip.auth.scheme": "Digest", + "sip.auth.uri.host": "192.168.1.2", + "sip.auth.uri.original": "sip:192.168.1.2", + "sip.auth.uri.scheme": "sip", + "sip.call_id": "578222729-4665d775@578222732-4665d772", + "sip.contact.uri.host": "sip.cybercity.dk", + "sip.contact.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "voi18062", + "sip.cseq.code": 75, + "sip.cseq.method": "REGISTER", + "sip.from.tag": "6bac55c", + "sip.from.uri.host": "sip.cybercity.dk", + "sip.from.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "voi18062", + "sip.max_forwards": 70, + "sip.method": "REGISTER", + "sip.to.uri.host": "sip.cybercity.dk", + "sip.to.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "voi18062", + "sip.type": "request", + "sip.uri.host": "sip.cybercity.dk", + "sip.uri.original": "sip:sip.cybercity.dk", + "sip.uri.scheme": "sip", + "sip.user_agent.original": "Nero SIPPS IP Phone Version 2.0.51.16", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport" + ], + "source.ip": "192.168.1.2", + "source.port": 5060, + "status": "OK", + "type": "sip", + "user.name": "voi18062" +} +``` + diff --git a/packetbeat/protos/sip/_meta/fields.yml b/packetbeat/protos/sip/_meta/fields.yml new file mode 100644 index 00000000000..fcbb1349dd1 --- /dev/null +++ b/packetbeat/protos/sip/_meta/fields.yml @@ -0,0 +1,238 @@ +- key: sip + title: "SIP" + description: SIP-specific event fields. + fields: + - name: sip + type: group + description: Information about SIP traffic. + fields: + - name: timestamp + type: date_nanos + description: Timestamp with nano second precision. + - name: code + type: keyword + description: Response status code. + - name: method + type: keyword + description: Request method. + - name: status + type: keyword + description: Response status phrase. + - name: type + type: keyword + description: Either request or response. + - name: version + type: keyword + description: SIP protocol version. + - name: uri.original + type: keyword + description: The original URI. + multi_fields: + - name: text + type: text + analyzer: simple + - name: uri.scheme + type: keyword + description: The URI scheme. + - name: uri.username + type: keyword + description: The URI user name. + - name: uri.host + type: keyword + description: The URI host. + - name: uri.port + type: keyword + description: The URI port. + - name: accept + type: keyword + description: Accept header value. + - name: allow + type: keyword + description: Allowed methods. + - name: call_id + type: keyword + description: Call ID. + - name: content_length + type: long + - name: content_type + type: keyword + - name: max_forwards + type: long + - name: supported + type: keyword + description: Supported methods. + - name: user_agent.original + type: keyword + multi_fields: + - name: text + type: text + analyzer: simple + - name: private.uri.original + type: keyword + description: Private original URI. + multi_fields: + - name: text + type: text + analyzer: simple + - name: private.uri.scheme + type: keyword + description: Private URI scheme. + - name: private.uri.username + type: keyword + description: Private URI user name. + - name: private.uri.host + type: keyword + description: Private URI host. + - name: private.uri.port + type: keyword + description: Private URI port. + - name: cseq.code + type: keyword + description: Sequence code. + - name: cseq.method + type: keyword + description: Sequence method. + - name: via.original + type: keyword + description: The original Via value. + multi_fields: + - name: text + type: text + analyzer: simple + - name: to.display_info + type: keyword + description: "To display info" + - name: to.uri.original + type: keyword + description: "To original URI" + multi_fields: + - name: text + type: text + analyzer: simple + - name: to.uri.scheme + type: keyword + description: "To URI scheme" + - name: to.uri.username + type: keyword + description: "To URI user name" + - name: to.uri.host + type: keyword + description: "To URI host" + - name: to.uri.port + type: keyword + description: "To URI port" + - name: to.tag + type: keyword + description: "To tag" + - name: from.display_info + type: keyword + description: "From display info" + - name: from.uri.original + type: keyword + description: "From original URI" + multi_fields: + - name: text + type: text + analyzer: simple + - name: from.uri.scheme + type: keyword + description: "From URI scheme" + - name: from.uri.username + type: keyword + description: "From URI user name" + - name: from.uri.host + type: keyword + description: "From URI host" + - name: from.uri.port + type: keyword + description: "From URI port" + - name: from.tag + type: keyword + description: "From tag" + - name: contact.display_info + type: keyword + description: "Contact display info" + - name: contact.uri.original + type: keyword + description: "Contact original URI" + multi_fields: + - name: text + type: text + analyzer: simple + - name: contact.uri.scheme + type: keyword + description: "Contat URI scheme" + - name: contact.uri.username + type: keyword + description: "Contact URI user name" + - name: contact.uri.host + type: keyword + description: "Contact URI host" + - name: contact.uri.port + type: keyword + description: "Contact URI port" + - name: contact.transport + type: keyword + description: "Contact transport" + - name: contact.line + type: keyword + description: "Contact line" + - name: contact.expires + type: keyword + description: "Contact expires" + - name: contact.q + type: keyword + description: "Contact Q" + - name: auth.scheme + type: keyword + description: "Auth scheme" + - name: auth.realm + type: keyword + description: "Auth realm" + - name: auth.uri.original + type: keyword + description: "Auth original URI" + multi_fields: + - name: text + type: text + analyzer: simple + - name: auth.uri.scheme + type: keyword + description: "Auth URI scheme" + - name: auth.uri.host + type: keyword + description: "Auth URI host" + - name: auth.uri.port + type: keyword + description: "Auth URI port" + - name: sdp.version + type: keyword + description: "SDP version" + - name: sdp.owner.username + type: keyword + description: "SDP owner user name" + - name: sdp.owner.session_id + type: keyword + description: "SDP owner session ID" + - name: sdp.owner.version + type: keyword + description: "SDP owner version" + - name: sdp.owner.ip + type: ip + description: "SDP owner IP" + - name: sdp.session.name + type: keyword + description: "SDP session name" + - name: sdp.connection.info + type: keyword + description: "SDP connection info" + - name: sdp.connection.address + type: keyword + description: "SDP connection address" + - name: sdp.body.original + type: keyword + description: "SDP original body" + multi_fields: + - name: text + type: text + analyzer: simple diff --git a/packetbeat/protos/sip/config.go b/packetbeat/protos/sip/config.go new file mode 100644 index 00000000000..58a92606e80 --- /dev/null +++ b/packetbeat/protos/sip/config.go @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sip + +import ( + cfg "github.com/elastic/beats/v7/packetbeat/config" + "github.com/elastic/beats/v7/packetbeat/protos" +) + +type config struct { + cfg.ProtocolCommon `config:",inline"` + ParseAuthorization bool `config:"parse_authorization"` + ParseBody bool `config:"parse_body"` + KeepOriginal bool `config:"keep_original"` +} + +var ( + defaultConfig = config{ + ProtocolCommon: cfg.ProtocolCommon{ + TransactionTimeout: protos.DefaultTransactionExpiration, + }, + ParseAuthorization: true, + ParseBody: true, + KeepOriginal: true, + } +) diff --git a/packetbeat/protos/sip/event.go b/packetbeat/protos/sip/event.go new file mode 100644 index 00000000000..fcb9112fe93 --- /dev/null +++ b/packetbeat/protos/sip/event.go @@ -0,0 +1,102 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sip + +import ( + "github.com/elastic/beats/v7/libbeat/common" +) + +// ProtocolFields contains SIP fields. +type ProtocolFields struct { + Timestamp int64 `ecs:"timestamp"` + Code int `ecs:"code"` + Method common.NetString `ecs:"method"` + Status common.NetString `ecs:"status"` + Type string `ecs:"type"` + Version string `ecs:"version"` + + URIOriginal common.NetString `ecs:"uri.original"` + URIScheme common.NetString `ecs:"uri.scheme"` + URIUsername common.NetString `ecs:"uri.username"` + URIHost common.NetString `ecs:"uri.host"` + URIPort int `ecs:"uri.port"` + + Accept common.NetString `ecs:"accept"` + Allow []string `ecs:"allow"` + CallID common.NetString `ecs:"call_id"` + ContentLength int `ecs:"content_length"` + ContentType common.NetString `ecs:"content_type"` + MaxForwards int `ecs:"max_forwards"` + Supported []string `ecs:"supported"` + UserAgentOriginal common.NetString `ecs:"user_agent.original"` + + PrivateURIOriginal common.NetString `ecs:"private.uri.original"` + PrivateURIScheme common.NetString `ecs:"private.uri.scheme"` + PrivateURIUsername common.NetString `ecs:"private.uri.username"` + PrivateURIHost common.NetString `ecs:"private.uri.host"` + PrivateURIPort int `ecs:"private.uri.port"` + + CseqCode int `ecs:"cseq.code"` + CseqMethod common.NetString `ecs:"cseq.method"` + + ViaOriginal []common.NetString `ecs:"via.original"` + + ToDisplayInfo common.NetString `ecs:"to.display_info"` + ToURIOriginal common.NetString `ecs:"to.uri.original"` + ToURIScheme common.NetString `ecs:"to.uri.scheme"` + ToURIUsername common.NetString `ecs:"to.uri.username"` + ToURIHost common.NetString `ecs:"to.uri.host"` + ToURIPort int `ecs:"to.uri.port"` + ToTag common.NetString `ecs:"to.tag"` + + FromDisplayInfo common.NetString `ecs:"from.display_info"` + FromURIOriginal common.NetString `ecs:"from.uri.original"` + FromURIScheme common.NetString `ecs:"from.uri.scheme"` + FromURIUsername common.NetString `ecs:"from.uri.username"` + FromURIHost common.NetString `ecs:"from.uri.host"` + FromURIPort int `ecs:"from.uri.port"` + FromTag common.NetString `ecs:"from.tag"` + + ContactDisplayInfo common.NetString `ecs:"contact.display_info"` + ContactURIOriginal common.NetString `ecs:"contact.uri.original"` + ContactURIScheme common.NetString `ecs:"contact.uri.scheme"` + ContactURIUsername common.NetString `ecs:"contact.uri.username"` + ContactURIHost common.NetString `ecs:"contact.uri.host"` + ContactURIPort int `ecs:"contact.uri.port"` + ContactTransport common.NetString `ecs:"contact.transport"` + ContactLine common.NetString `ecs:"contact.line"` + ContactExpires int `ecs:"contact.expires"` + ContactQ float64 `ecs:"contact.q"` + + AuthScheme common.NetString `ecs:"auth.scheme"` + AuthRealm common.NetString `ecs:"auth.realm"` + AuthURIOriginal common.NetString `ecs:"auth.uri.original"` + AuthURIScheme common.NetString `ecs:"auth.uri.scheme"` + AuthURIHost common.NetString `ecs:"auth.uri.host"` + AuthURIPort int `ecs:"auth.uri.port"` + + SDPVersion string `ecs:"sdp.version"` + SDPOwnerUsername common.NetString `ecs:"sdp.owner.username"` + SDPOwnerSessID common.NetString `ecs:"sdp.owner.session_id"` + SDPOwnerVersion common.NetString `ecs:"sdp.owner.version"` + SDPOwnerIP common.NetString `ecs:"sdp.owner.ip"` + SDPSessName common.NetString `ecs:"sdp.session.name"` + SDPConnInfo common.NetString `ecs:"sdp.connection.info"` + SDPConnAddr common.NetString `ecs:"sdp.connection.address"` + SDPBodyOriginal common.NetString `ecs:"sdp.body.original"` +} diff --git a/packetbeat/protos/sip/fields.go b/packetbeat/protos/sip/fields.go new file mode 100644 index 00000000000..87c93ab0cad --- /dev/null +++ b/packetbeat/protos/sip/fields.go @@ -0,0 +1,36 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package sip + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("packetbeat", "sip", asset.ModuleFieldsPri, AssetSip); err != nil { + panic(err) + } +} + +// AssetSip returns asset data. +// This is the base64 encoded gzipped contents of protos/sip. +func AssetSip() string { + return "eJzEmcFy4jgQhu95ii7u8QNwm5rsVnHLhsxeKY3dxqqRJUdqQ9in35KxFGEkZqRUhZzAcX+/1Pz9ozKP8AtPazB8eAAgTgLXsNpunlcPAA2aWvOBuJJr2G6eH82ANW95DXhASdByFI2pHmB+tX4AAHgEyXp0SPtHpwHXsNdqdFcuyBvZKt0z+wbYTzWS1QLSrG15Xc0VocKHBvEeDbHecZ1Wwwh3kkll/D8uJF9dHRw5dWDvBIO1kg0MGmtuuJLVQqtWDS5kfuHpqHQT13hBMyhpEAwxGs1Uv2T2SJ1q8qhvIxqaK5e8s9RnVjl0mpmrdVpQDvUvTh1q0PNilX151lmSD6hts3Pg1h2DVqRqJVz9EjtqXinN91wykcN+7RBcHfx42VT+tn4UxHeXNrxoEb5TcNmpXV1mkonTf6jtgPSDwMjCTd1hn9Vvu+wfLxs4V8aaMRrU9l0J1dZOqBi4U4ZKoLYuxhuULuLZuiWP1TUOWbRvUwV0yBrUcGBivNo0E0Ids5i2AJt5ZM1VrjAhdjwrBL4zIWDzdJ1QklDSTqDcU7cACiX3ift/O90+rtj7rlX6yHSzDJkI3oyD/VAwa2tbV5RqlzXjju1R0p8O+BdN7qD5gRFWpdHzfK6/b/yEm8iPIbeFdBSF/JJIChWSsRSK5MZTKBCLqJCdG1UhOxZXtcG3KveYsbXfsbLG6PliIuYfMjwzfso4cPb5L9d/ObvM1y/zOKmq4WYQ7LTjslU5O1i9KphrwdaurtGl42/R4eiv7tCWsqm3K/+Y+ERLSobdgf2gJ9i5M+64ti6BzB1th7R1ESSxfS6N2H4JarXqy637t1b9TfNO+GL7Tvi7Gtivv8DC0+rTJvboIht7eNLInp9tZc+Omdljs+3ssTFDT9hcS0/EiKntIZTVVO7r72fATWs7kWJ3O5G7GjzcRYHHpz3QDZeH/CKjuy7d9Hqokm33UCHm+BCebfoQHvO9g5Nm0hTTfXUKL7gsa7stTEHxfeAasx4Nee5cm0K/FUH/WeLYSF2Jrb+N1CUMPSE1MtHnE6eyKLA4RCbuXRPEr7+0z+nw8Ojsmfbg2EB7bPa8eWxs1kwzVAWPHlfbp2f3yDGGVEeJuiw7LXkqTyfnh4JBY5eQ+dgo0JgBsHlKi5T256zw2y7x5UP74EIKev5dYsmbN1MV9dx1ItXwWkmJtS2oso8mlv9RHz2ZLCRY02g0eTG9UJkRMaGfqjmVRdf0EbjkspgviK7/AwAA///sesAq" +} diff --git a/packetbeat/protos/sip/parser.go b/packetbeat/protos/sip/parser.go new file mode 100644 index 00000000000..55e66045e95 --- /dev/null +++ b/packetbeat/protos/sip/parser.go @@ -0,0 +1,469 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sip + +import ( + "bytes" + "errors" + "fmt" + "time" + "unicode" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/streambuf" + "github.com/elastic/beats/v7/packetbeat/procs" + "github.com/elastic/beats/v7/packetbeat/protos" +) + +// Http Message +type message struct { + ts time.Time + hasContentLength bool + headerOffset int + + isRequest bool + ipPortTuple common.IPPortTuple + cmdlineTuple *common.ProcessTuple + + // Info + requestURI common.NetString + method common.NetString + statusCode uint16 + statusPhrase common.NetString + version version + + // Headers + contentLength int + contentType common.NetString + userAgent common.NetString + to common.NetString + from common.NetString + cseq common.NetString + callID common.NetString + maxForwards int + via []common.NetString + allow []string + supported []string + + headers map[string][]common.NetString + size uint64 + + firstLine []byte + rawHeaders []byte + body []byte + rawData []byte +} + +type version struct { + major uint8 + minor uint8 +} + +func (v version) String() string { + return fmt.Sprintf("%d.%d", v.major, v.minor) +} + +type parserState uint8 + +const ( + stateStart parserState = iota + stateHeaders + stateBody +) + +type parser struct { +} + +type parsingInfo struct { + tuple *common.IPPortTuple + + data []byte + + parseOffset int + state parserState + bodyReceived int + + pkt *protos.Packet +} + +var ( + constCRLF = []byte("\r\n") + constSIPVersion = []byte("SIP/") + nameContentLength = []byte("content-length") + nameContentType = []byte("content-type") + nameUserAgent = []byte("user-agent") + nameTo = []byte("to") + nameFrom = []byte("from") + nameCseq = []byte("cseq") + nameCallID = []byte("call-id") + nameMaxForwards = []byte("max-forwards") + nameAllow = []byte("allow") + nameSupported = []byte("supported") + nameVia = []byte("via") +) + +func newParser() *parser { + return &parser{} +} + +func (parser *parser) parse(pi *parsingInfo) (*message, error) { + m := &message{ + ts: pi.pkt.Ts, + ipPortTuple: pi.pkt.Tuple, + cmdlineTuple: procs.ProcWatcher.FindProcessesTupleTCP(&pi.pkt.Tuple), + rawData: pi.data, + } + for pi.parseOffset < len(pi.data) { + switch pi.state { + case stateStart: + if err := parser.parseSIPLine(pi, m); err != nil { + return m, err + } + case stateHeaders: + if err := parser.parseHeaders(pi, m); err != nil { + return m, err + } + case stateBody: + parser.parseBody(pi, m) + } + } + return m, nil +} + +func (*parser) parseSIPLine(pi *parsingInfo, m *message) error { + // ignore any CRLF appearing before the start-line (RFC3261 7.5) + pi.data = bytes.TrimLeft(pi.data[pi.parseOffset:], string(constCRLF)) + + i := bytes.Index(pi.data[pi.parseOffset:], constCRLF) + if i == -1 { + return errors.New("not found expected CRLF") + } + + // Very basic tests on the first line. Just to check that + // we have what looks as a SIP message + var ( + version []byte + err error + ) + + fline := pi.data[pi.parseOffset:i] + if len(fline) < 16 { // minimum line will be "SIP/2.0 XXX OK\r\n" + if isDebug { + debugf("First line too small") + } + return errors.New("first line too small") + } + + m.firstLine = fline + + if bytes.Equal(fline[0:4], constSIPVersion) { + // RESPONSE + m.isRequest = false + version = fline[4:7] + m.statusCode, m.statusPhrase, err = parseResponseStatus(fline[8:]) + if err != nil { + if isDebug { + debugf("Failed to understand SIP response status: %s", fline[8:]) + } + return errors.New("failed to parse response status") + } + + if isDebug { + debugf("SIP status_code=%d, status_phrase=%s", m.statusCode, m.statusPhrase) + } + } else { + // REQUEST + afterMethodIdx := bytes.IndexFunc(fline, unicode.IsSpace) + afterRequestURIIdx := bytes.LastIndexFunc(fline, unicode.IsSpace) + + // Make sure we have the VERB + URI + SIP_VERSION + if afterMethodIdx == -1 || afterRequestURIIdx == -1 || afterMethodIdx == afterRequestURIIdx { + if isDebug { + debugf("Couldn't understand SIP request: %s", fline) + } + return errors.New("failed to parse SIP request") + } + + m.method = common.NetString(fline[:afterMethodIdx]) + m.requestURI = common.NetString(fline[afterMethodIdx+1 : afterRequestURIIdx]) + + versionIdx := afterRequestURIIdx + len(constSIPVersion) + 1 + if len(fline) > versionIdx && bytes.Equal(fline[afterRequestURIIdx+1:versionIdx], constSIPVersion) { + m.isRequest = true + version = fline[versionIdx:] + } else { + if isDebug { + debugf("Couldn't understand SIP version: %s", fline) + } + return errors.New("failed to parse SIP version") + } + } + + m.version.major, m.version.minor, err = parseVersion(version) + if err != nil { + if isDebug { + debugf(err.Error(), version) + } + return err + } + if isDebug { + debugf("SIP version %d.%d", m.version.major, m.version.minor) + } + + // ok so far + pi.parseOffset = i + 2 + m.headerOffset = pi.parseOffset + pi.state = stateHeaders + + return nil +} + +func parseResponseStatus(s []byte) (uint16, []byte, error) { + if isDebug { + debugf("parseResponseStatus: %s", s) + } + + var phrase []byte + p := bytes.IndexByte(s, ' ') + if p == -1 { + p = len(s) + } else { + phrase = s[p+1:] + } + statusCode, err := parseInt(s[0:p]) + if err != nil { + return 0, nil, fmt.Errorf("Unable to parse status code from [%s]", s) + } + return uint16(statusCode), phrase, nil +} + +func parseVersion(s []byte) (uint8, uint8, error) { + if len(s) < 3 { + return 0, 0, errors.New("Invalid version") + } + + major := s[0] - '0' + minor := s[2] - '0' + + return uint8(major), uint8(minor), nil +} + +func (parser *parser) parseHeaders(pi *parsingInfo, m *message) error { + // check if it isn't headers end yet with /r/n/r/n + if !(len(pi.data)-pi.parseOffset >= 2 && + bytes.Equal(pi.data[pi.parseOffset:pi.parseOffset+2], constCRLF)) { + offset, err := parser.parseHeader(m, pi.data[pi.parseOffset:]) + if err != nil { + return err + } + + pi.parseOffset += offset + + return nil + } + + m.size = uint64(pi.parseOffset + 2) + m.rawHeaders = pi.data[:m.size] + pi.data = pi.data[m.size:] + pi.parseOffset = 0 + + if m.contentLength == 0 && (m.isRequest || m.hasContentLength) { + if isDebug { + debugf("Empty content length, ignore body") + } + return nil + } + + if isDebug { + debugf("Read body") + } + + pi.state = stateBody + + return nil +} + +func (parser *parser) parseHeader(m *message, data []byte) (int, error) { + if m.headers == nil { + m.headers = make(map[string][]common.NetString) + } + + i := bytes.Index(data, []byte(":")) + if i == -1 { + // Expected \":\" in headers. Assuming incomplete + if isDebug { + debugf("ignoring incomplete header %s", data) + } + return len(data), nil + } + + // enabled if required. Allocs for parameters slow down parser big times + if isDetailed { + detailedf("Data: %s", data) + detailedf("Header: %s", data[:i]) + } + + // skip folding line + for p := i + 1; p < len(data); { + q := bytes.Index(data[p:], constCRLF) + if q == -1 { + if isDebug { + debugf("ignoring incomplete header %s", data) + } + return len(data), nil + } + + p += q + if len(data) > p && (data[p+1] == ' ' || data[p+1] == '\t') { + p = p + 2 + continue + } + + headerName := getExpandedHeaderName(bytes.ToLower(data[:i])) + headerVal := bytes.TrimSpace(data[i+1 : p]) + if isDebug { + debugf("Header: '%s' Value: '%s'\n", data[:i], headerVal) + } + + // Headers we need for parsing. Make sure we always + // capture their value + switch { + case bytes.Equal(headerName, nameMaxForwards): + m.maxForwards, _ = parseInt(headerVal) + case bytes.Equal(headerName, nameContentLength): + m.contentLength, _ = parseInt(headerVal) + m.hasContentLength = true + case bytes.Equal(headerName, nameContentType): + m.contentType = headerVal + case bytes.Equal(headerName, nameUserAgent): + m.userAgent = headerVal + case bytes.Equal(headerName, nameTo): + m.to = headerVal + case bytes.Equal(headerName, nameFrom): + m.from = headerVal + case bytes.Equal(headerName, nameCseq): + m.cseq = headerVal + case bytes.Equal(headerName, nameCallID): + m.callID = headerVal + case bytes.Equal(headerName, nameAllow): + m.allow = parseCommaSeparatedList(headerVal) + case bytes.Equal(headerName, nameSupported): + m.supported = parseCommaSeparatedList(headerVal) + case bytes.Equal(headerName, nameVia): + m.via = append(m.via, headerVal) + } + + m.headers[string(headerName)] = append( + m.headers[string(headerName)], + headerVal, + ) + + return p + 2, nil + } + + return len(data), nil +} + +func parseCommaSeparatedList(s common.NetString) (list []string) { + values := bytes.Split(s, []byte(",")) + list = make([]string, len(values)) + for idx := range values { + list[idx] = string(bytes.ToLower(bytes.Trim(values[idx], " "))) + } + return list +} + +func (*parser) parseBody(pi *parsingInfo, m *message) { + nbytes := len(pi.data) + if nbytes >= m.contentLength-pi.bodyReceived { + wanted := m.contentLength - pi.bodyReceived + m.body = append(m.body, pi.data[:wanted]...) + pi.bodyReceived = m.contentLength + m.size += uint64(wanted) + pi.data = pi.data[wanted:] + } else { + m.body = append(m.body, pi.data...) + pi.data = nil + pi.bodyReceived += nbytes + m.size += uint64(nbytes) + if isDebug { + debugf("bodyReceived: %d", pi.bodyReceived) + } + } +} + +func (m *message) getEndpoints() (src *common.Endpoint, dst *common.Endpoint) { + source, destination := common.MakeEndpointPair(m.ipPortTuple.BaseTuple, m.cmdlineTuple) + src, dst = &source, &destination + return src, dst +} + +func parseInt(line []byte) (int, error) { + buf := streambuf.NewFixed(line) + i, err := buf.IntASCII(false) + return int(i), err + // TODO: is it an error if 'buf.Len() != 0 {}' ? +} + +func getExpandedHeaderName(n []byte) []byte { + if len(n) > 1 { + return n + } + switch string(n) { + // referfenced by https://www.iana.org/assignments/sip-parameters/sip-parameters.xhtml + case "a": + return []byte("accept-contact") //[RFC3841] + case "b": + return []byte("referred-by") //[RFC3892] + case "c": + return []byte("content-type") //[RFC3261] + case "d": + return []byte("request-disposition") //[RFC3841] + case "e": + return []byte("content-encoding") //[RFC3261] + case "f": + return []byte("from") //[RFC3261] + case "i": + return []byte("call-id") //[RFC3261] + case "j": + return []byte("reject-contact") //[RFC3841] + case "k": + return []byte("supported") //[RFC3261] + case "l": + return []byte("content-length") //[RFC3261] + case "m": + return []byte("contact") //[RFC3261] + case "o": + return []byte("event") //[RFC666)5] [RFC6446] + case "r": + return []byte("refer-to") //[RFC3515] + case "s": + return []byte("subject") //[RFC3261] + case "t": + return []byte("to") //[RFC3261] + case "u": + return []byte("allow-events") //[RFC6665] + case "v": + return []byte("via") //[RFC326)1] [RFC7118] + case "x": + return []byte("session-expires") //[RFC4028] + case "y": + return []byte("identity") //[RFC8224] + } + return n +} diff --git a/packetbeat/protos/sip/plugin.go b/packetbeat/protos/sip/plugin.go new file mode 100644 index 00000000000..14b56aeda45 --- /dev/null +++ b/packetbeat/protos/sip/plugin.go @@ -0,0 +1,636 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package sip + +import ( + "bytes" + "fmt" + "strconv" + "strings" + "time" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/packetbeat/pb" + "github.com/elastic/beats/v7/packetbeat/protos" +) + +var ( + debugf = logp.MakeDebug("sip") + detailedf = logp.MakeDebug("sipdetailed") +) + +// SIP application level protocol analyser plugin. +type plugin struct { + // config + ports []int + parseAuthorization bool + parseBody bool + keepOriginal bool + + results protos.Reporter +} + +var ( + isDebug = false + isDetailed = false +) + +func init() { + protos.Register("sip", New) +} + +func New( + testMode bool, + results protos.Reporter, + cfg *common.Config, +) (protos.Plugin, error) { + cfgwarn.Beta("packetbeat SIP protocol is used") + + p := &plugin{} + config := defaultConfig + if !testMode { + if err := cfg.Unpack(&config); err != nil { + return nil, err + } + } + + if err := p.init(results, &config); err != nil { + return nil, err + } + return p, nil +} + +// Init initializes the HTTP protocol analyser. +func (p *plugin) init(results protos.Reporter, config *config) error { + p.setFromConfig(config) + + isDebug = logp.IsDebug("sip") + isDetailed = logp.IsDebug("sipdetailed") + p.results = results + return nil +} + +func (p *plugin) setFromConfig(config *config) { + p.ports = config.Ports + p.keepOriginal = config.KeepOriginal + p.parseAuthorization = config.ParseAuthorization + p.parseBody = config.ParseBody +} + +func (p *plugin) GetPorts() []int { + return p.ports +} + +func (p *plugin) ParseUDP(pkt *protos.Packet) { + defer logp.Recover("SIP ParseUDP exception") + + if err := p.doParse(pkt); err != nil { + debugf("error: %s", err) + } +} + +func (p *plugin) doParse(pkt *protos.Packet) error { + if isDetailed { + detailedf("Payload received: [%s]", pkt.Payload) + } + + parser := newParser() + + pi := newParsingInfo(pkt) + m, err := parser.parse(pi) + if err != nil { + return err + } + + evt, err := p.buildEvent(m, pkt) + if err != nil { + return err + } + + p.publish(*evt) + + return nil +} + +func (p *plugin) publish(evt beat.Event) { + if p.results != nil { + p.results(evt) + } +} + +func newParsingInfo(pkt *protos.Packet) *parsingInfo { + return &parsingInfo{ + tuple: &pkt.Tuple, + data: pkt.Payload, + pkt: pkt, + } +} + +func (p *plugin) buildEvent(m *message, pkt *protos.Packet) (*beat.Event, error) { + status := common.OK_STATUS + if m.statusCode >= 400 { + status = common.ERROR_STATUS + } + + evt, pbf := pb.NewBeatEvent(m.ts) + fields := evt.Fields + fields["type"] = "sip" + fields["status"] = status + + var sipFields ProtocolFields + sipFields.Timestamp = time.Now().UnixNano() + if m.isRequest { + populateRequestFields(m, pbf, &sipFields) + } else { + populateResponseFields(m, &sipFields) + } + + p.populateHeadersFields(m, evt, pbf, &sipFields) + + if p.parseBody { + populateBodyFields(m, pbf, &sipFields) + } + + pbf.Network.IANANumber = "17" + pbf.Network.Application = "sip" + pbf.Network.Protocol = "sip" + pbf.Network.Transport = "udp" + + src, dst := m.getEndpoints() + pbf.SetSource(src) + pbf.SetDestination(dst) + + p.populateEventFields(m, pbf, sipFields) + + if err := pb.MarshalStruct(evt.Fields, "sip", sipFields); err != nil { + return nil, err + } + + return &evt, nil +} + +func populateRequestFields(m *message, pbf *pb.Fields, fields *ProtocolFields) { + fields.Type = "request" + fields.Method = bytes.ToUpper(m.method) + fields.URIOriginal = m.requestURI + scheme, username, host, port, _ := parseURI(fields.URIOriginal) + fields.URIScheme = scheme + fields.URIHost = host + if !bytes.Equal(username, []byte(" ")) && !bytes.Equal(username, []byte("-")) { + fields.URIUsername = username + pbf.AddUser(string(username)) + } + fields.URIPort = port + fields.Version = m.version.String() + pbf.AddHost(string(host)) +} + +func populateResponseFields(m *message, fields *ProtocolFields) { + fields.Type = "response" + fields.Code = int(m.statusCode) + fields.Status = m.statusPhrase + fields.Version = m.version.String() +} + +func (p *plugin) populateHeadersFields(m *message, evt beat.Event, pbf *pb.Fields, fields *ProtocolFields) { + fields.Allow = m.allow + fields.CallID = m.callID + fields.ContentLength = m.contentLength + fields.ContentType = bytes.ToLower(m.contentType) + fields.MaxForwards = m.maxForwards + fields.Supported = m.supported + fields.UserAgentOriginal = m.userAgent + fields.ViaOriginal = m.via + + privateURI, found := m.headers["p-associated-uri"] + if found && len(privateURI) > 0 { + scheme, username, host, port, _ := parseURI(privateURI[0]) + fields.PrivateURIOriginal = privateURI[0] + fields.PrivateURIScheme = scheme + fields.PrivateURIHost = host + if !bytes.Equal(username, []byte(" ")) && !bytes.Equal(username, []byte("-")) { + fields.PrivateURIUsername = username + pbf.AddUser(string(username)) + } + fields.PrivateURIPort = port + pbf.AddHost(string(host)) + } + + if accept, found := m.headers["accept"]; found && len(accept) > 0 { + fields.Accept = bytes.ToLower(accept[0]) + } + + cseqParts := bytes.Split(m.cseq, []byte(" ")) + if len(cseqParts) == 2 { + fields.CseqCode, _ = strconv.Atoi(string(cseqParts[0])) + fields.CseqMethod = bytes.ToUpper(cseqParts[1]) + } + + populateFromFields(m, pbf, fields) + + populateToFields(m, pbf, fields) + + populateContactFields(m, pbf, fields) + + if p.parseAuthorization { + populateAuthFields(m, evt, pbf, fields) + } +} + +func populateFromFields(m *message, pbf *pb.Fields, fields *ProtocolFields) { + if len(m.from) > 0 { + displayInfo, uri, params := parseFromToContact(m.from) + fields.FromDisplayInfo = displayInfo + fields.FromTag = params["tag"] + scheme, username, host, port, _ := parseURI(uri) + fields.FromURIOriginal = uri + fields.FromURIScheme = scheme + fields.FromURIHost = host + if !bytes.Equal(username, []byte(" ")) && !bytes.Equal(username, []byte("-")) { + fields.FromURIUsername = username + pbf.AddUser(string(username)) + } + fields.FromURIPort = port + pbf.AddHost(string(host)) + } +} + +func populateToFields(m *message, pbf *pb.Fields, fields *ProtocolFields) { + if len(m.to) > 0 { + displayInfo, uri, params := parseFromToContact(m.to) + fields.ToDisplayInfo = displayInfo + fields.ToTag = params["tag"] + scheme, username, host, port, _ := parseURI(uri) + fields.ToURIOriginal = uri + fields.ToURIScheme = scheme + fields.ToURIHost = host + if !bytes.Equal(username, []byte(" ")) && !bytes.Equal(username, []byte("-")) { + fields.ToURIUsername = username + pbf.AddUser(string(username)) + } + fields.ToURIPort = port + pbf.AddHost(string(host)) + } +} + +func populateContactFields(m *message, pbf *pb.Fields, fields *ProtocolFields) { + if contact, found := m.headers["contact"]; found && len(contact) > 0 { + displayInfo, uri, params := parseFromToContact(m.to) + fields.ContactDisplayInfo = displayInfo + fields.ContactExpires, _ = strconv.Atoi(string(params["expires"])) + fields.ContactQ, _ = strconv.ParseFloat(string(params["q"]), 64) + scheme, username, host, port, urlparams := parseURI(uri) + fields.ContactURIOriginal = uri + fields.ContactURIScheme = scheme + fields.ContactURIHost = host + if !bytes.Equal(username, []byte(" ")) && !bytes.Equal(username, []byte("-")) { + fields.ContactURIUsername = username + pbf.AddUser(string(username)) + } + fields.ContactURIPort = port + fields.ContactLine = urlparams["line"] + fields.ContactTransport = bytes.ToLower(urlparams["transport"]) + pbf.AddHost(string(host)) + } +} + +func (p *plugin) populateEventFields(m *message, pbf *pb.Fields, fields ProtocolFields) { + pbf.Event.Kind = "event" + pbf.Event.Type = []string{"info"} + pbf.Event.Dataset = "sip" + pbf.Event.Sequence = int64(fields.CseqCode) + + // TODO: Get these values from body + pbf.Event.Start = m.ts + pbf.Event.End = m.ts + // + + if p.keepOriginal { + pbf.Event.Original = string(m.rawData) + } + + pbf.Event.Category = []string{"network", "protocol"} + if _, found := m.headers["authorization"]; found { + pbf.Event.Category = append(pbf.Event.Category, "authentication") + } + + pbf.Event.Action = func() string { + if m.isRequest { + return fmt.Sprintf("sip-%s", strings.ToLower(string(m.method))) + } + return fmt.Sprintf("sip-%s", strings.ToLower(string(fields.CseqMethod))) + }() + + pbf.Event.Outcome = func() string { + switch { + case m.statusCode < 200: + return "" + case m.statusCode > 299: + return "failure" + } + return "success" + }() + + pbf.Event.Reason = string(fields.Status) +} + +func populateAuthFields(m *message, evt beat.Event, pbf *pb.Fields, fields *ProtocolFields) { + auths, found := m.headers["authorization"] + if !found || len(auths) == 0 { + if isDetailed { + detailedf("sip packet without authorization header") + } + return + } + + if isDetailed { + detailedf("sip packet with authorization header") + } + + auth := bytes.TrimSpace(auths[0]) + pos := bytes.IndexByte(auth, ' ') + if pos == -1 { + if isDebug { + debugf("malformed authorization header: missing scheme") + } + return + } + + fields.AuthScheme = auth[:pos] + + pos += 1 + for _, param := range bytes.Split(auth[pos:], []byte(",")) { + kv := bytes.SplitN(param, []byte("="), 2) + if len(kv) != 2 { + continue + } + kv[1] = bytes.Trim(kv[1], "'\" \t") + switch string(bytes.ToLower(bytes.TrimSpace(kv[0]))) { + case "realm": + fields.AuthRealm = kv[1] + case "username": + username := string(kv[1]) + if username != "" && username != "-" { + _, _ = evt.Fields.Put("user.name", username) + pbf.AddUser(username) + } + case "uri": + scheme, _, host, port, _ := parseURI(kv[1]) + fields.AuthURIOriginal = kv[1] + fields.AuthURIScheme = scheme + fields.AuthURIHost = host + fields.AuthURIPort = port + } + } +} + +var constSDPContentType = []byte("application/sdp") + +func populateBodyFields(m *message, pbf *pb.Fields, fields *ProtocolFields) { + if !m.hasContentLength { + return + } + + if !bytes.Equal(m.contentType, constSDPContentType) { + if isDebug { + debugf("body content-type: %s is not supported", m.contentType) + } + return + } + + if _, found := m.headers["content-encoding"]; found { + if isDebug { + debugf("body decoding is not supported yet if content-endcoding is present") + } + return + } + + fields.SDPBodyOriginal = m.body + + var isInMedia bool + for _, line := range bytes.Split(m.body, []byte("\r\n")) { + kv := bytes.SplitN(line, []byte("="), 2) + if len(kv) != 2 { + continue + } + + kv[1] = bytes.TrimSpace(kv[1]) + ch := string(bytes.ToLower(bytes.TrimSpace(kv[0]))) + switch ch { + case "v": + fields.SDPVersion = string(kv[1]) + case "o": + var pos int + if kv[1][pos] == '"' { + endUserPos := bytes.IndexByte(kv[1][pos+1:], '"') + if !bytes.Equal(kv[1][pos+1:endUserPos], []byte("-")) { + fields.SDPOwnerUsername = kv[1][pos+1 : endUserPos] + } + pos = endUserPos + 1 + } + nParts := func() int { + if pos == 0 { + return 4 + } + return 3 // already have user + }() + parts := bytes.SplitN(kv[1][pos:], []byte(" "), nParts) + if len(parts) != nParts { + if isDebug { + debugf("malformed owner SDP line") + } + continue + } + if nParts == 4 { + if !bytes.Equal(parts[0], []byte("-")) { + fields.SDPOwnerUsername = parts[0] + } + parts = parts[1:] + } + fields.SDPOwnerSessID = parts[0] + fields.SDPOwnerVersion = parts[1] + fields.SDPOwnerIP = func() common.NetString { + p := bytes.Split(parts[2], []byte(" ")) + return p[len(p)-1] + }() + pbf.AddUser(string(fields.SDPOwnerUsername)) + pbf.AddIP(string(fields.SDPOwnerIP)) + case "s": + if !bytes.Equal(kv[1], []byte("-")) { + fields.SDPSessName = kv[1] + } + case "c": + if isInMedia { + continue + } + fields.SDPConnInfo = kv[1] + fields.SDPConnAddr = func() common.NetString { + p := bytes.Split(kv[1], []byte(" ")) + return p[len(p)-1] + }() + pbf.AddHost(string(fields.SDPConnAddr)) + case "m": + isInMedia = true + // TODO + case "i", "u", "e", "p", "b", "t", "r", "z", "k", "a": + // TODO + } + } +} + +func parseFromToContact(fromTo common.NetString) (displayInfo, uri common.NetString, params map[string]common.NetString) { + params = make(map[string]common.NetString) + + fromTo = bytes.TrimSpace(fromTo) + + var uriIsWrapped bool + pos := func() int { + // look for the beginning of a url wrapped in <...> + if pos := bytes.IndexByte(fromTo, '<'); pos > -1 { + uriIsWrapped = true + return pos + } + // if there is no < char, it means there is no display info, and + // that the url starts from the beginning + // https://tools.ietf.org/html/rfc3261#section-20.10 + return 0 + }() + + displayInfo = bytes.Trim(fromTo[:pos], "'\"\t ") + + endURIPos := func() int { + if uriIsWrapped { + return bytes.IndexByte(fromTo, '>') + } + return bytes.IndexByte(fromTo, ';') + }() + + // not wrapped and no header params + if endURIPos == -1 { + uri = fromTo[pos:] + return displayInfo, uri, params + } + + // if wrapped, we want to get over the < char + if uriIsWrapped { + pos += 1 + } + + // if wrapped, we will get the string between <...> + // if not wrapped, we will get the value before the header params (until ;) + uri = fromTo[pos:endURIPos] + + // parse the header params + pos = endURIPos + 1 + for _, param := range bytes.Split(fromTo[pos:], []byte(";")) { + kv := bytes.SplitN(param, []byte("="), 2) + if len(kv) != 2 { + continue + } + params[string(bytes.ToLower(bytes.TrimSpace(kv[0])))] = kv[1] + } + + return displayInfo, uri, params +} + +func parseURI(uri common.NetString) (scheme, username, host common.NetString, port int, params map[string]common.NetString) { + var ( + prevChar rune + inIPv6 bool + idx int + hasParams bool + ) + uri = bytes.TrimSpace(uri) + prevChar = ' ' + pos := -1 + ppos := -1 + epos := len(uri) + + params = make(map[string]common.NetString) +loop: + for idx = 0; idx < len(uri); idx++ { + curChar := rune(uri[idx]) + + switch { + case idx == 0: + colonIdx := bytes.Index(uri, []byte(":")) + if colonIdx == -1 { + break loop + } + scheme = uri[:colonIdx] + idx += colonIdx + pos = idx + 1 + + case curChar == '[' && prevChar != '\\': + inIPv6 = true + + case curChar == ']' && prevChar != '\\': + inIPv6 = false + + case curChar == ';' && prevChar != '\\': + // we found end of URI + hasParams = true + epos = idx + break loop + + default: + // select wich part + switch curChar { + case '@': + if len(host) > 0 { + pos = ppos + host = nil + } + username = uri[pos:idx] + ppos = pos + pos = idx + 1 + case ':': + if !inIPv6 { + host = uri[pos:idx] + ppos = pos + pos = idx + 1 + } + } + } + + prevChar = curChar + } + + if pos > 0 && epos <= len(uri) && pos <= epos { + if len(host) == 0 { + host = bytes.TrimSpace(uri[pos:epos]) + } else { + port, _ = strconv.Atoi(string(bytes.TrimSpace(uri[pos:epos]))) + } + } + + if hasParams { + for _, param := range bytes.Split(uri[epos+1:], []byte(";")) { + kv := bytes.Split(param, []byte("=")) + if len(kv) != 2 { + continue + } + params[string(bytes.ToLower(bytes.TrimSpace(kv[0])))] = kv[1] + } + } + + return scheme, username, host, port, params +} diff --git a/packetbeat/protos/sip/plugin_test.go b/packetbeat/protos/sip/plugin_test.go new file mode 100644 index 00000000000..d8c09f5b307 --- /dev/null +++ b/packetbeat/protos/sip/plugin_test.go @@ -0,0 +1,174 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !integration + +package sip + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/packetbeat/protos" +) + +func TestParseURI(t *testing.T) { + scheme, username, host, port, params := parseURI(common.NetString("sip:test@10.0.2.15:5060")) + assert.Equal(t, common.NetString("sip"), scheme) + assert.Equal(t, common.NetString("test"), username) + assert.Equal(t, common.NetString("10.0.2.15"), host) + assert.Equal(t, map[string]common.NetString{}, params) + assert.Equal(t, 5060, port) + + scheme, username, host, port, params = parseURI(common.NetString("sips:test@10.0.2.15:5061 ; transport=udp")) + assert.Equal(t, common.NetString("sips"), scheme) + assert.Equal(t, common.NetString("test"), username) + assert.Equal(t, common.NetString("10.0.2.15"), host) + assert.Equal(t, common.NetString("udp"), params["transport"]) + assert.Equal(t, 5061, port) + + scheme, username, host, port, params = parseURI(common.NetString("mailto:192.168.0.2")) + assert.Equal(t, common.NetString("mailto"), scheme) + assert.Equal(t, common.NetString(nil), username) + assert.Equal(t, common.NetString("192.168.0.2"), host) + assert.Equal(t, map[string]common.NetString{}, params) + assert.Equal(t, 0, port) +} + +func TestParseFromTo(t *testing.T) { + // To + displayInfo, uri, params := parseFromToContact(common.NetString("test ;tag=QvN921")) + assert.Equal(t, common.NetString("test"), displayInfo) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), uri) + assert.Equal(t, common.NetString("QvN921"), params["tag"]) + displayInfo, uri, params = parseFromToContact(common.NetString("test ")) + assert.Equal(t, common.NetString("test"), displayInfo) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), uri) + assert.Equal(t, common.NetString(nil), params["tag"]) + + // From + displayInfo, uri, params = parseFromToContact(common.NetString("\"PCMU/8000\" ;tag=1")) + assert.Equal(t, common.NetString("PCMU/8000"), displayInfo) + assert.Equal(t, common.NetString("sip:sipp@10.0.2.15:5060"), uri) + assert.Equal(t, common.NetString("1"), params["tag"]) + displayInfo, uri, params = parseFromToContact(common.NetString(" \"Matthew Hodgson\" ;tag=5c7cdb68")) + assert.Equal(t, common.NetString("Matthew Hodgson"), displayInfo) + assert.Equal(t, common.NetString("sip:matthew@mxtelecom.com"), uri) + assert.Equal(t, common.NetString("5c7cdb68"), params["tag"]) + displayInfo, uri, params = parseFromToContact(common.NetString(";tag=5c7cdb68")) + assert.Equal(t, common.NetString(nil), displayInfo) + assert.Equal(t, common.NetString("sip:matthew@mxtelecom.com"), uri) + assert.Equal(t, common.NetString("5c7cdb68"), params["tag"]) + displayInfo, uri, params = parseFromToContact(common.NetString("")) + assert.Equal(t, common.NetString(nil), displayInfo) + assert.Equal(t, common.NetString("sip:matthew@mxtelecom.com"), uri) + assert.Equal(t, common.NetString(nil), params["tag"]) + + // Contact + displayInfo, uri, _ = parseFromToContact(common.NetString(" ")) + assert.Equal(t, common.NetString(nil), displayInfo) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060;transport=udp"), uri) + displayInfo, uri, params = parseFromToContact(common.NetString(";expires=1200;q=0.500")) + assert.Equal(t, common.NetString(nil), displayInfo) + assert.Equal(t, common.NetString("sip:voi18062@192.168.1.2:5060;line=aca6b97ca3f5e51a"), uri) + assert.Equal(t, common.NetString("1200"), params["expires"]) + assert.Equal(t, common.NetString("0.500"), params["q"]) + displayInfo, uri, params = parseFromToContact(common.NetString(" \"Mr. Watson\" ;q=0.7; expires=3600")) + assert.Equal(t, common.NetString("Mr. Watson"), displayInfo) + assert.Equal(t, common.NetString("sip:watson@worcester.bell-telephone.com"), uri) + assert.Equal(t, common.NetString("3600"), params["expires"]) + assert.Equal(t, common.NetString("0.7"), params["q"]) + displayInfo, uri, params = parseFromToContact(common.NetString(" \"Mr. Watson\" ;q=0.1")) + assert.Equal(t, common.NetString("Mr. Watson"), displayInfo) + assert.Equal(t, common.NetString("mailto:watson@bell-telephone.com"), uri) + assert.Equal(t, common.NetString("0.1"), params["q"]) + + // url is not bounded by <...> + displayInfo, uri, params = parseFromToContact(common.NetString(" sip:test@10.0.2.15:5060;transport=udp")) + assert.Equal(t, common.NetString(nil), displayInfo) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), uri) + assert.Equal(t, common.NetString("udp"), params["transport"]) +} + +func TestParseUDP(t *testing.T) { + gotEvent := new(beat.Event) + reporter := func(evt beat.Event) { + gotEvent = &evt + } + const data = "INVITE sip:test@10.0.2.15:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0\r\nFrom: \"DVI4/8000\" ;tag=1\r\nTo: test \r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 1 INVITE\r\nContact: sip:sipp@10.0.2.20:5060\r\nMax-Forwards: 70\r\nContent-Type: application/sdp\r\nContent-Length: 123\r\n\r\nv=0\r\no=- 42 42 IN IP4 10.0.2.20\r\ns=-\r\nc=IN IP4 10.0.2.20\r\nt=0 0\r\nm=audio 6000 RTP/AVP 5\r\na=rtpmap:5 DVI4/8000\r\na=recvonly\r\n" + p, _ := New(true, reporter, nil) + plugin := p.(*plugin) + plugin.ParseUDP(&protos.Packet{ + Ts: time.Now(), + Tuple: common.IPPortTuple{}, + Payload: []byte(data), + }) + fields := *gotEvent + + assert.Equal(t, common.NetString("1-2187@10.0.2.20"), getVal(fields, "sip.call_id")) + assert.Equal(t, common.NetString("test"), getVal(fields, "sip.contact.display_info")) + assert.Equal(t, common.NetString("10.0.2.15"), getVal(fields, "sip.contact.uri.host")) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), getVal(fields, "sip.contact.uri.original")) + assert.Equal(t, 5060, getVal(fields, "sip.contact.uri.port")) + assert.Equal(t, common.NetString("sip"), getVal(fields, "sip.contact.uri.scheme")) + assert.Equal(t, common.NetString("test"), getVal(fields, "sip.contact.uri.username")) + assert.Equal(t, 123, getVal(fields, "sip.content_length")) + assert.Equal(t, common.NetString("application/sdp"), getVal(fields, "sip.content_type")) + assert.Equal(t, 1, getVal(fields, "sip.cseq.code")) + assert.Equal(t, common.NetString("INVITE"), getVal(fields, "sip.cseq.method")) + assert.Equal(t, common.NetString("DVI4/8000"), getVal(fields, "sip.from.display_info")) + assert.Equal(t, common.NetString("1"), getVal(fields, "sip.from.tag")) + assert.Equal(t, common.NetString("10.0.2.20"), getVal(fields, "sip.from.uri.host")) + assert.Equal(t, common.NetString("sip:sipp@10.0.2.20:5060"), getVal(fields, "sip.from.uri.original")) + assert.Equal(t, 5060, getVal(fields, "sip.from.uri.port")) + assert.Equal(t, common.NetString("sip"), getVal(fields, "sip.from.uri.scheme")) + assert.Equal(t, common.NetString("sipp"), getVal(fields, "sip.from.uri.username")) + assert.Equal(t, 70, getVal(fields, "sip.max_forwards")) + assert.Equal(t, common.NetString("INVITE"), getVal(fields, "sip.method")) + assert.Equal(t, common.NetString("10.0.2.20"), getVal(fields, "sip.sdp.connection.address")) + assert.Equal(t, common.NetString("IN IP4 10.0.2.20"), getVal(fields, "sip.sdp.connection.info")) + assert.Equal(t, common.NetString("10.0.2.20"), getVal(fields, "sip.sdp.owner.ip")) + assert.Equal(t, common.NetString("42"), getVal(fields, "sip.sdp.owner.session_id")) + assert.Equal(t, common.NetString("42"), getVal(fields, "sip.sdp.owner.version")) + assert.Equal(t, nil, getVal(fields, "sip.sdp.owner.username")) + assert.Equal(t, nil, getVal(fields, "sip.sdp.session.name")) + assert.Equal(t, "0", getVal(fields, "sip.sdp.version")) + assert.Equal(t, common.NetString("test"), getVal(fields, "sip.to.display_info")) + assert.Equal(t, nil, getVal(fields, "sip.to.tag")) + assert.Equal(t, common.NetString("10.0.2.15"), getVal(fields, "sip.to.uri.host")) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), getVal(fields, "sip.to.uri.original")) + assert.Equal(t, 5060, getVal(fields, "sip.to.uri.port")) + assert.Equal(t, common.NetString("sip"), getVal(fields, "sip.to.uri.scheme")) + assert.Equal(t, common.NetString("test"), getVal(fields, "sip.to.uri.username")) + assert.Equal(t, "request", getVal(fields, "sip.type")) + assert.Equal(t, common.NetString("10.0.2.15"), getVal(fields, "sip.uri.host")) + assert.Equal(t, common.NetString("sip:test@10.0.2.15:5060"), getVal(fields, "sip.uri.original")) + assert.Equal(t, 5060, getVal(fields, "sip.uri.port")) + assert.Equal(t, common.NetString("sip"), getVal(fields, "sip.uri.scheme")) + assert.Equal(t, common.NetString("test"), getVal(fields, "sip.uri.username")) + assert.Equal(t, "2.0", getVal(fields, "sip.version")) + assert.EqualValues(t, []common.NetString{common.NetString("SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0")}, getVal(fields, "sip.via.original")) +} + +func getVal(f beat.Event, k string) interface{} { + v, _ := f.GetValue(k) + return v +} diff --git a/packetbeat/tests/system/config/golden-tests.yml b/packetbeat/tests/system/config/golden-tests.yml index 6d49ce9c221..42ad0c746d2 100644 --- a/packetbeat/tests/system/config/golden-tests.yml +++ b/packetbeat/tests/system/config/golden-tests.yml @@ -27,3 +27,11 @@ test_cases: - name: TLS 1.3 pcap: pcaps/tls-version-13.pcap config: {} + + - name: SIP + pcap: pcaps/sip.pcap + config: {} + + - name: SIP Authenticated Register + pcap: pcaps/sip_authenticated_register.pcap + config: {} diff --git a/packetbeat/tests/system/config/packetbeat.yml.j2 b/packetbeat/tests/system/config/packetbeat.yml.j2 index b687f6f0402..7b253d8ec2c 100644 --- a/packetbeat/tests/system/config/packetbeat.yml.j2 +++ b/packetbeat/tests/system/config/packetbeat.yml.j2 @@ -148,6 +148,9 @@ packetbeat.protocols: {% if mongodb_max_docs is not none %} max_docs: {{mongodb_max_docs}}{% endif %} {% if mongodb_max_doc_length is not none %} max_doc_length: {{mongodb_max_doc_length}}{% endif %} +- type: sip + ports: [{{ sip_ports|default([5060])|join(", ") }}] + {% if procs_enabled %} #=========================== Monitored processes ============================== diff --git a/packetbeat/tests/system/golden/sip-expected.json b/packetbeat/tests/system/golden/sip-expected.json new file mode 100644 index 00000000000..37d95d715a7 --- /dev/null +++ b/packetbeat/tests/system/golden/sip-expected.json @@ -0,0 +1,668 @@ +[ + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "INVITE sip:test@10.0.2.15:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0\r\nFrom: \"DVI4/8000\" ;tag=1\r\nTo: test \r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 1 INVITE\r\nContact: sip:sipp@10.0.2.20:5060\r\nMax-Forwards: 70\r\nContent-Type: application/sdp\r\nContent-Length: 123\r\n\r\nv=0\r\no=- 42 42 IN IP4 10.0.2.20\r\ns=-\r\nc=IN IP4 10.0.2.20\r\nt=0 0\r\nm=audio 6000 RTP/AVP 5\r\na=rtpmap:5 DVI4/8000\r\na=recvonly\r\n", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2187@10.0.2.20", + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.content_length": 123, + "sip.content_type": "application/sdp", + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/8000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.max_forwards": 70, + "sip.method": "INVITE", + "sip.sdp.body.original": "v=0\r\no=- 42 42 IN IP4 10.0.2.20\r\ns=-\r\nc=IN IP4 10.0.2.20\r\nt=0 0\r\nm=audio 6000 RTP/AVP 5\r\na=rtpmap:5 DVI4/8000\r\na=recvonly\r\n", + "sip.sdp.connection.address": "10.0.2.20", + "sip.sdp.connection.info": "IN IP4 10.0.2.20", + "sip.sdp.owner.ip": "10.0.2.20", + "sip.sdp.owner.session_id": "42", + "sip.sdp.owner.version": "42", + "sip.sdp.version": "0", + "sip.to.display_info": "test", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "request", + "sip.uri.host": "10.0.2.15", + "sip.uri.original": "sip:test@10.0.2.15:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "test", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0\r\nFrom: \"DVI4/8000\" ;tag=1\r\nTo: test \r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 1 INVITE\r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nContent-Length: 0\r\n\r\n", + "event.reason": "Trying", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.call_id": "1-2187@10.0.2.20", + "sip.code": 100, + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/8000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.status": "Trying", + "sip.to.display_info": "test", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "response", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-0" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-ack", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "ACK sip:test@10.0.2.15:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-5\r\nFrom: \"DVI4/8000\" ;tag=1\r\nTo: test ;tag=e2jv529vDZ3eQ\r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 1 ACK\r\nContact: sip:sipp@10.0.2.20:5060\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r\n", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2187@10.0.2.20", + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.cseq.code": 1, + "sip.cseq.method": "ACK", + "sip.from.display_info": "DVI4/8000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.max_forwards": 70, + "sip.method": "ACK", + "sip.to.display_info": "test", + "sip.to.tag": "e2jv529vDZ3eQ", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "request", + "sip.uri.host": "10.0.2.15", + "sip.uri.original": "sip:test@10.0.2.15:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "test", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2187-1-5" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-bye", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "BYE sip:sipp@10.0.2.20:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKDQ7XK6BBH57ya\r\nMax-Forwards: 70\r\nFrom: test ;tag=e2jv529vDZ3eQ\r\nTo: \"DVI4/8000\" ;tag=1\r\nCall-ID: 1-2187@10.0.2.20\r\nCSeq: 99750433 BYE\r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nAllow: INVITE, ACK, BYE, CANCEL, OPTIONS, MESSAGE, INFO, UPDATE, REGISTER, REFER, NOTIFY, PUBLISH, SUBSCRIBE\r\nSupported: timer, path, replaces\r\nReason: Q.850;cause=16;text=\"NORMAL_CLEARING\"\r\nContent-Length: 0\r\n\r\n", + "event.sequence": 99750433, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.allow": [ + "invite", + "ack", + "bye", + "cancel", + "options", + "message", + "info", + "update", + "register", + "refer", + "notify", + "publish", + "subscribe" + ], + "sip.call_id": "1-2187@10.0.2.20", + "sip.cseq.code": 99750433, + "sip.cseq.method": "BYE", + "sip.from.display_info": "test", + "sip.from.tag": "e2jv529vDZ3eQ", + "sip.from.uri.host": "10.0.2.15", + "sip.from.uri.original": "sip:test@10.0.2.15:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "test", + "sip.max_forwards": 70, + "sip.method": "BYE", + "sip.supported": [ + "timer", + "path", + "replaces" + ], + "sip.to.display_info": "DVI4/8000", + "sip.to.tag": "1", + "sip.to.uri.host": "10.0.2.20", + "sip.to.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "sipp", + "sip.type": "request", + "sip.uri.host": "10.0.2.20", + "sip.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "sipp", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKDQ7XK6BBH57ya" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "INVITE sip:test@10.0.2.15:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0\r\nFrom: \"DVI4/16000\" ;tag=1\r\nTo: test \r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 1 INVITE\r\nContact: sip:sipp@10.0.2.20:5060\r\nMax-Forwards: 70\r\nContent-Type: application/sdp\r\nContent-Length: 124\r\n\r\nv=0\r\no=- 42 42 IN IP4 10.0.2.20\r\ns=-\r\nc=IN IP4 10.0.2.20\r\nt=0 0\r\nm=audio 6000 RTP/AVP 6\r\na=rtpmap:6 DVI4/16000\r\na=recvonly\r\n", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2189@10.0.2.20", + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.content_length": 124, + "sip.content_type": "application/sdp", + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/16000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.max_forwards": 70, + "sip.method": "INVITE", + "sip.sdp.body.original": "v=0\r\no=- 42 42 IN IP4 10.0.2.20\r\ns=-\r\nc=IN IP4 10.0.2.20\r\nt=0 0\r\nm=audio 6000 RTP/AVP 6\r\na=rtpmap:6 DVI4/16000\r\na=recvonly\r\n", + "sip.sdp.connection.address": "10.0.2.20", + "sip.sdp.connection.info": "IN IP4 10.0.2.20", + "sip.sdp.owner.ip": "10.0.2.20", + "sip.sdp.owner.session_id": "42", + "sip.sdp.owner.version": "42", + "sip.sdp.version": "0", + "sip.to.display_info": "test", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "request", + "sip.uri.host": "10.0.2.15", + "sip.uri.original": "sip:test@10.0.2.15:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "test", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-invite", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 100 Trying\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0\r\nFrom: \"DVI4/16000\" ;tag=1\r\nTo: test \r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 1 INVITE\r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nContent-Length: 0\r\n\r\n", + "event.reason": "Trying", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.call_id": "1-2189@10.0.2.20", + "sip.code": 100, + "sip.cseq.code": 1, + "sip.cseq.method": "INVITE", + "sip.from.display_info": "DVI4/16000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.status": "Trying", + "sip.to.display_info": "test", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "response", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-0" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.20", + "client.port": 5060, + "destination.ip": "10.0.2.15", + "destination.port": 5060, + "event.action": "sip-ack", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "ACK sip:test@10.0.2.15:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-5\r\nFrom: \"DVI4/16000\" ;tag=1\r\nTo: test ;tag=FBcN7Xt0a8S1j\r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 1 ACK\r\nContact: sip:sipp@10.0.2.20:5060\r\nMax-Forwards: 70\r\nContent-Length: 0\r\n\r\n", + "event.sequence": 1, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.ip": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.user": [ + "test", + "sipp" + ], + "server.ip": "10.0.2.15", + "server.port": 5060, + "sip.call_id": "1-2189@10.0.2.20", + "sip.contact.display_info": "test", + "sip.contact.uri.host": "10.0.2.15", + "sip.contact.uri.original": "sip:test@10.0.2.15:5060", + "sip.contact.uri.port": 5060, + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "test", + "sip.cseq.code": 1, + "sip.cseq.method": "ACK", + "sip.from.display_info": "DVI4/16000", + "sip.from.tag": "1", + "sip.from.uri.host": "10.0.2.20", + "sip.from.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "sipp", + "sip.max_forwards": 70, + "sip.method": "ACK", + "sip.to.display_info": "test", + "sip.to.tag": "FBcN7Xt0a8S1j", + "sip.to.uri.host": "10.0.2.15", + "sip.to.uri.original": "sip:test@10.0.2.15:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "test", + "sip.type": "request", + "sip.uri.host": "10.0.2.15", + "sip.uri.original": "sip:test@10.0.2.15:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "test", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.20:5060;branch=z9hG4bK-2189-1-5" + ], + "source.ip": "10.0.2.20", + "source.port": 5060, + "status": "OK", + "type": "sip" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "10.0.2.15", + "client.port": 5060, + "destination.ip": "10.0.2.20", + "destination.port": 5060, + "event.action": "sip-bye", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "BYE sip:sipp@10.0.2.20:5060 SIP/2.0\r\nVia: SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKe00pN1veeeyHp\r\nMax-Forwards: 70\r\nFrom: test ;tag=FBcN7Xt0a8S1j\r\nTo: \"DVI4/16000\" ;tag=1\r\nCall-ID: 1-2189@10.0.2.20\r\nCSeq: 99750437 BYE\r\nUser-Agent: FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit\r\nAllow: INVITE, ACK, BYE, CANCEL, OPTIONS, MESSAGE, INFO, UPDATE, REGISTER, REFER, NOTIFY, PUBLISH, SUBSCRIBE\r\nSupported: timer, path, replaces\r\nReason: Q.850;cause=16;text=\"NORMAL_CLEARING\"\r\nContent-Length: 0\r\n\r\n", + "event.sequence": 99750437, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:xDRQZvk3ErEhBDslXv1c6EKI804=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "10.0.2.20", + "10.0.2.15" + ], + "related.ip": [ + "10.0.2.15", + "10.0.2.20" + ], + "related.user": [ + "sipp", + "test" + ], + "server.ip": "10.0.2.20", + "server.port": 5060, + "sip.allow": [ + "invite", + "ack", + "bye", + "cancel", + "options", + "message", + "info", + "update", + "register", + "refer", + "notify", + "publish", + "subscribe" + ], + "sip.call_id": "1-2189@10.0.2.20", + "sip.cseq.code": 99750437, + "sip.cseq.method": "BYE", + "sip.from.display_info": "test", + "sip.from.tag": "FBcN7Xt0a8S1j", + "sip.from.uri.host": "10.0.2.15", + "sip.from.uri.original": "sip:test@10.0.2.15:5060", + "sip.from.uri.port": 5060, + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "test", + "sip.max_forwards": 70, + "sip.method": "BYE", + "sip.supported": [ + "timer", + "path", + "replaces" + ], + "sip.to.display_info": "DVI4/16000", + "sip.to.tag": "1", + "sip.to.uri.host": "10.0.2.20", + "sip.to.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.to.uri.port": 5060, + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "sipp", + "sip.type": "request", + "sip.uri.host": "10.0.2.20", + "sip.uri.original": "sip:sipp@10.0.2.20:5060", + "sip.uri.port": 5060, + "sip.uri.scheme": "sip", + "sip.uri.username": "sipp", + "sip.user_agent.original": "FreeSWITCH-mod_sofia/1.6.12-20-b91a0a6~64bit", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 10.0.2.15;rport;branch=z9hG4bKe00pN1veeeyHp" + ], + "source.ip": "10.0.2.15", + "source.port": 5060, + "status": "OK", + "type": "sip" + } +] \ No newline at end of file diff --git a/packetbeat/tests/system/golden/sip_authenticated_register-expected.json b/packetbeat/tests/system/golden/sip_authenticated_register-expected.json new file mode 100644 index 00000000000..133792cc157 --- /dev/null +++ b/packetbeat/tests/system/golden/sip_authenticated_register-expected.json @@ -0,0 +1,142 @@ +[ + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "192.168.1.2", + "client.port": 5060, + "destination.ip": "212.242.33.35", + "destination.port": 5060, + "event.action": "sip-register", + "event.category": [ + "network", + "protocol", + "authentication" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "REGISTER sip:sip.cybercity.dk SIP/2.0\r\nVia: SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport\r\nFrom: ;tag=6bac55c\r\nTo: \r\nCall-ID: 578222729-4665d775@578222732-4665d772\r\nContact: ;expires=1200;q=0.500\r\nExpires: 1200\r\nCSeq: 75 REGISTER\r\nContent-Length: 0\r\nAuthorization: Digest username=\"voi18062\",realm=\"sip.cybercity.dk\",uri=\"sip:192.168.1.2\",nonce=\"1701b22972b90f440c3e4eb250842bb\",opaque=\"1701a1351f70795\",nc=\"00000001\",response=\"79a0543188495d288c9ebbe0c881abdc\"\r\nMax-Forwards: 70\r\nUser-Agent: Nero SIPPS IP Phone Version 2.0.51.16\r\n\r\n", + "event.sequence": 75, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:dOa61R2NaaJsJlcFAiMIiyXX+Kk=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "sip.cybercity.dk" + ], + "related.ip": [ + "192.168.1.2", + "212.242.33.35" + ], + "related.user": [ + "voi18062" + ], + "server.ip": "212.242.33.35", + "server.port": 5060, + "sip.auth.realm": "sip.cybercity.dk", + "sip.auth.scheme": "Digest", + "sip.auth.uri.host": "192.168.1.2", + "sip.auth.uri.original": "sip:192.168.1.2", + "sip.auth.uri.scheme": "sip", + "sip.call_id": "578222729-4665d775@578222732-4665d772", + "sip.contact.uri.host": "sip.cybercity.dk", + "sip.contact.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.contact.uri.scheme": "sip", + "sip.contact.uri.username": "voi18062", + "sip.cseq.code": 75, + "sip.cseq.method": "REGISTER", + "sip.from.tag": "6bac55c", + "sip.from.uri.host": "sip.cybercity.dk", + "sip.from.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "voi18062", + "sip.max_forwards": 70, + "sip.method": "REGISTER", + "sip.to.uri.host": "sip.cybercity.dk", + "sip.to.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "voi18062", + "sip.type": "request", + "sip.uri.host": "sip.cybercity.dk", + "sip.uri.original": "sip:sip.cybercity.dk", + "sip.uri.scheme": "sip", + "sip.user_agent.original": "Nero SIPPS IP Phone Version 2.0.51.16", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 192.168.1.2;branch=z9hG4bKnp112903503-43a64480192.168.1.2;rport" + ], + "source.ip": "192.168.1.2", + "source.port": 5060, + "status": "OK", + "type": "sip", + "user.name": "voi18062" + }, + { + "@metadata.beat": "packetbeat", + "@metadata.type": "_doc", + "client.ip": "212.242.33.35", + "client.port": 5060, + "destination.ip": "192.168.1.2", + "destination.port": 5060, + "event.action": "sip-register", + "event.category": [ + "network", + "protocol" + ], + "event.dataset": "sip", + "event.duration": 0, + "event.kind": "event", + "event.original": "SIP/2.0 100 Trying\r\nCall-ID: 578222729-4665d775@578222732-4665d772\r\nCSeq: 75 REGISTER\r\nFrom: ;tag=6bac55c\r\nTo: \r\nVia: SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060;branch=z9hG4bKnp112903503-43a64480192.168.1.2\r\nContent-Length: 0\r\n\r\n", + "event.reason": "Trying", + "event.sequence": 75, + "event.type": [ + "info" + ], + "network.application": "sip", + "network.community_id": "1:dOa61R2NaaJsJlcFAiMIiyXX+Kk=", + "network.iana_number": "17", + "network.protocol": "sip", + "network.transport": "udp", + "network.type": "ipv4", + "related.hosts": [ + "sip.cybercity.dk" + ], + "related.ip": [ + "212.242.33.35", + "192.168.1.2" + ], + "related.user": [ + "voi18062" + ], + "server.ip": "192.168.1.2", + "server.port": 5060, + "sip.call_id": "578222729-4665d775@578222732-4665d772", + "sip.code": 100, + "sip.cseq.code": 75, + "sip.cseq.method": "REGISTER", + "sip.from.tag": "6bac55c", + "sip.from.uri.host": "sip.cybercity.dk", + "sip.from.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.from.uri.scheme": "sip", + "sip.from.uri.username": "voi18062", + "sip.status": "Trying", + "sip.to.uri.host": "sip.cybercity.dk", + "sip.to.uri.original": "sip:voi18062@sip.cybercity.dk", + "sip.to.uri.scheme": "sip", + "sip.to.uri.username": "voi18062", + "sip.type": "response", + "sip.version": "2.0", + "sip.via.original": [ + "SIP/2.0/UDP 192.168.1.2;received=80.230.219.70;rport=5060;branch=z9hG4bKnp112903503-43a64480192.168.1.2" + ], + "source.ip": "212.242.33.35", + "source.port": 5060, + "status": "OK", + "type": "sip" + } +] \ No newline at end of file diff --git a/packetbeat/tests/system/pcaps/sip.pcap b/packetbeat/tests/system/pcaps/sip.pcap new file mode 100644 index 00000000000..7ec19fb525b Binary files /dev/null and b/packetbeat/tests/system/pcaps/sip.pcap differ diff --git a/packetbeat/tests/system/pcaps/sip_authenticated_register.pcap b/packetbeat/tests/system/pcaps/sip_authenticated_register.pcap new file mode 100644 index 00000000000..25f10e5cf88 Binary files /dev/null and b/packetbeat/tests/system/pcaps/sip_authenticated_register.pcap differ diff --git a/packetbeat/tests/system/test_0099_golden_files.py b/packetbeat/tests/system/test_0099_golden_files.py index 0acc7ca8ad1..e01bc955600 100644 --- a/packetbeat/tests/system/test_0099_golden_files.py +++ b/packetbeat/tests/system/test_0099_golden_files.py @@ -79,6 +79,9 @@ def clean_keys(obj): # Network direction is populated based on local-IPs which is misleading # when reading from a pcap and leads to inconsistent results. "network.direction", + + # module specific + "sip.timestamp", ] for key in keys: if key in obj: diff --git a/winlogbeat/Jenkinsfile.yml b/winlogbeat/Jenkinsfile.yml index 74eb55586d0..82ce7912d3b 100644 --- a/winlogbeat/Jenkinsfile.yml +++ b/winlogbeat/Jenkinsfile.yml @@ -19,3 +19,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test winlogbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/winlogbeat/cmd/root.go b/winlogbeat/cmd/root.go index 2cd26a9fe8e..988ac6f9f6c 100644 --- a/winlogbeat/cmd/root.go +++ b/winlogbeat/cmd/root.go @@ -37,7 +37,7 @@ const ( Name = "winlogbeat" // ecsVersion specifies the version of ECS that Winlogbeat is implementing. - ecsVersion = "1.5.0" + ecsVersion = "1.6.0" ) // withECSVersion is a modifier that adds ecs.version to events. diff --git a/winlogbeat/docs/fields.asciidoc b/winlogbeat/docs/fields.asciidoc index d0b1a0a1473..cc84c0fad8a 100644 --- a/winlogbeat/docs/fields.asciidoc +++ b/winlogbeat/docs/fields.asciidoc @@ -220,8 +220,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + diff --git a/winlogbeat/eventlog/eventlogging.go b/winlogbeat/eventlog/eventlogging.go index 963797264b2..01465d933fc 100644 --- a/winlogbeat/eventlog/eventlogging.go +++ b/winlogbeat/eventlog/eventlogging.go @@ -40,9 +40,6 @@ const ( eventLoggingAPIName = "eventlogging" ) -var eventLoggingConfigKeys = common.MakeStringSet(append(commonConfigKeys, - "ignore_older", "read_buffer_size", "format_buffer_size")...) - type eventLoggingConfig struct { ConfigCommon `config:",inline"` IgnoreOlder time.Duration `config:"ignore_older"` @@ -284,7 +281,7 @@ func newEventLogging(options *common.Config) (EventLog, error) { ReadBufferSize: win.MaxEventBufferSize, FormatBufferSize: win.MaxFormatMessageBufferSize, } - if err := readConfig(options, &c, eventLoggingConfigKeys); err != nil { + if err := readConfig(options, &c); err != nil { return nil, err } diff --git a/winlogbeat/eventlog/factory.go b/winlogbeat/eventlog/factory.go index f66c158b2f2..9ee8e0d144f 100644 --- a/winlogbeat/eventlog/factory.go +++ b/winlogbeat/eventlog/factory.go @@ -22,14 +22,9 @@ import ( "sort" "strings" - "github.com/joeshaw/multierror" - "github.com/elastic/beats/v7/libbeat/common" ) -var commonConfigKeys = []string{"type", "api", "name", "fields", "fields_under_root", - "tags", "processors", "index", "id", "meta", "revision"} - // ConfigCommon is the common configuration data used to instantiate a new // EventLog. Each implementation is free to support additional configuration // options. @@ -42,33 +37,18 @@ type validator interface { Validate() error } -func readConfig( - c *common.Config, - config interface{}, - validKeys common.StringSet, -) error { +func readConfig(c *common.Config, config interface{}) error { if err := c.Unpack(config); err != nil { return fmt.Errorf("failed unpacking config. %v", err) } - var errs multierror.Errors - if len(validKeys) > 0 { - // Check for invalid keys. - for _, k := range c.GetFields() { - if !validKeys.Has(k) { - errs = append(errs, fmt.Errorf("invalid event log key '%s' "+ - "found. Valid keys are %s", k, strings.Join(validKeys.ToSlice(), ", "))) - } - } - } - if v, ok := config.(validator); ok { if err := v.Validate(); err != nil { - errs = append(errs, err) + return err } } - return errs.Err() + return nil } // Producer produces a new event log instance for reading event log records. @@ -114,7 +94,7 @@ func New(options *common.Config) (EventLog, error) { } var config ConfigCommon - if err := readConfig(options, &config, nil); err != nil { + if err := readConfig(options, &config); err != nil { return nil, err } diff --git a/winlogbeat/eventlog/wineventlog.go b/winlogbeat/eventlog/wineventlog.go index db00f239974..7ee6c62cf18 100644 --- a/winlogbeat/eventlog/wineventlog.go +++ b/winlogbeat/eventlog/wineventlog.go @@ -47,10 +47,6 @@ const ( winEventLogAPIName = "wineventlog" ) -var winEventLogConfigKeys = common.MakeStringSet(append(commonConfigKeys, - "batch_read_size", "ignore_older", "include_xml", "event_id", "forwarded", - "level", "provider", "no_more_events")...) - type winEventLogConfig struct { ConfigCommon `config:",inline"` BatchReadSize int `config:"batch_read_size"` // Maximum number of events that Read will return. @@ -366,7 +362,7 @@ func (l *winEventLog) buildRecordFromXML(x []byte, recoveredErr error) (Record, // using the Windows Event Log. func newWinEventLog(options *common.Config) (EventLog, error) { c := defaultWinEventLogConfig - if err := readConfig(options, &c, winEventLogConfigKeys); err != nil { + if err := readConfig(options, &c); err != nil { return nil, err } diff --git a/winlogbeat/eventlog/wineventlog_expirimental.go b/winlogbeat/eventlog/wineventlog_expirimental.go index 5952aad0f5a..301b2cf4f0b 100644 --- a/winlogbeat/eventlog/wineventlog_expirimental.go +++ b/winlogbeat/eventlog/wineventlog_expirimental.go @@ -243,7 +243,7 @@ func newWinEventLogExp(options *common.Config) (EventLog, error) { cfgwarn.Experimental("The %s event log reader is experimental.", winEventLogExpAPIName) c := winEventLogConfig{BatchReadSize: 512} - if err := readConfig(options, &c, winEventLogConfigKeys); err != nil { + if err := readConfig(options, &c); err != nil { return nil, err } diff --git a/winlogbeat/tests/system/test_eventlogging.py b/winlogbeat/tests/system/test_eventlogging.py index dd763104fa5..0d486656054 100644 --- a/winlogbeat/tests/system/test_eventlogging.py +++ b/winlogbeat/tests/system/test_eventlogging.py @@ -156,25 +156,6 @@ def test_ignore_older(self): self.assertEqual(evts[0]["winlog.event_id"], 10) self.assertEqual(evts[0]["event.code"], 10) - def test_unknown_eventlog_config(self): - """ - eventlogging - Unknown config parameter - """ - self.render_config_template( - event_logs=[ - { - "name": self.providerName, - "api": self.api, - "event_id": "10, 12", - "level": "info", - "provider": ["me"], - "include_xml": True, - } - ] - ) - self.start_beat().check_wait(exit_code=1) - assert self.log_contains("4 errors: invalid event log key") - def test_utf16_characters(self): """ eventlogging - UTF-16 characters diff --git a/winlogbeat/tests/system/test_wineventlog.py b/winlogbeat/tests/system/test_wineventlog.py index ea390475279..363e90edbd2 100644 --- a/winlogbeat/tests/system/test_wineventlog.py +++ b/winlogbeat/tests/system/test_wineventlog.py @@ -311,23 +311,6 @@ def test_query_multi_param(self): self.assertTrue(len(evts), 1) self.assertEqual(evts[0]["message"], "selected") - def test_unknown_eventlog_config(self): - """ - wineventlog - Unknown config parameter - """ - self.render_config_template( - event_logs=[ - { - "name": self.providerName, - "api": self.api, - "forwarded": False, - "invalid": "garbage"} - ] - ) - self.start_beat().check_wait(exit_code=1) - assert self.log_contains( - "1 error: invalid event log key 'invalid' found.") - def test_utf16_characters(self): """ wineventlog - UTF-16 characters diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 85535e8c920..5850d9947b4 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -46,3 +46,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test auditbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/auditbeat/module/system/socket/events.go b/x-pack/auditbeat/module/system/socket/events.go index 76d91a8db24..c1937cb0302 100644 --- a/x-pack/auditbeat/module/system/socket/events.go +++ b/x-pack/auditbeat/module/system/socket/events.go @@ -872,8 +872,8 @@ type execveCall struct { creds *commitCreds } -func (e *execveCall) getProcess() process { - p := process{ +func (e *execveCall) getProcess() *process { + p := &process{ pid: e.Meta.PID, path: readCString(e.Path[:]), created: kernelTime(e.Meta.Timestamp), diff --git a/x-pack/auditbeat/module/system/socket/socket_linux.go b/x-pack/auditbeat/module/system/socket/socket_linux.go index 78fdd8ae4ca..11f8a22289e 100644 --- a/x-pack/auditbeat/module/system/socket/socket_linux.go +++ b/x-pack/auditbeat/module/system/socket/socket_linux.go @@ -158,7 +158,7 @@ func (m *MetricSet) Run(r mb.PushReporterV2) { } else { for _, p := range procs { if i, err := p.Info(); err == nil { - process := process{ + process := &process{ name: i.Name, pid: uint32(i.PID), args: i.Args, diff --git a/x-pack/auditbeat/module/system/socket/state.go b/x-pack/auditbeat/module/system/socket/state.go index c7b3ac761a7..fa612b56e1a 100644 --- a/x-pack/auditbeat/module/system/socket/state.go +++ b/x-pack/auditbeat/module/system/socket/state.go @@ -214,6 +214,9 @@ func (f *flow) Timestamp() time.Time { } type process struct { + // RWMutex is used to arbitrate reads and writes to resolvedDomains. + sync.RWMutex + pid uint32 name, path string args []string @@ -229,6 +232,8 @@ type process struct { } func (p *process) addTransaction(tr dns.Transaction) { + p.Lock() + defer p.Unlock() if p.resolvedDomains == nil { p.resolvedDomains = make(map[string]string) } @@ -239,6 +244,8 @@ func (p *process) addTransaction(tr dns.Transaction) { // ResolveIP returns the domain associated with the given IP. func (p *process) ResolveIP(ip net.IP) (domain string, found bool) { + p.RLock() + defer p.RUnlock() domain, found = p.resolvedDomains[ip.String()] return } @@ -542,13 +549,13 @@ func (s *state) ExpireOlder() { s.dns.CleanUp() } -func (s *state) CreateProcess(p process) error { +func (s *state) CreateProcess(p *process) error { if p.pid == 0 { return errors.New("can't create process with PID 0") } s.Lock() defer s.Unlock() - s.processes[p.pid] = &p + s.processes[p.pid] = p if p.createdTime == (time.Time{}) { p.createdTime = s.kernTimestampToTime(p.created) } diff --git a/x-pack/auditbeat/module/system/socket/state_test.go b/x-pack/auditbeat/module/system/socket/state_test.go index 75b73a374d1..89ba0cde9db 100644 --- a/x-pack/auditbeat/module/system/socket/state_test.go +++ b/x-pack/auditbeat/module/system/socket/state_test.go @@ -11,6 +11,7 @@ import ( "fmt" "net" "os" + "sync" "testing" "time" @@ -835,3 +836,28 @@ func TestSocketReuse(t *testing.T) { } assert.Len(t, flows, 1) } + +func TestProcessDNSRace(t *testing.T) { + p := new(process) + var wg sync.WaitGroup + wg.Add(2) + address := func(i byte) net.IP { return net.IPv4(172, 16, 0, i) } + go func() { + for i := byte(255); i > 0; i-- { + p.addTransaction(dns.Transaction{ + Client: net.UDPAddr{IP: net.IPv4(10, 20, 30, 40)}, + Server: net.UDPAddr{IP: net.IPv4(10, 20, 30, 41)}, + Domain: "example.net", + Addresses: []net.IP{address(i)}, + }) + } + wg.Done() + }() + go func() { + for i := byte(255); i > 0; i-- { + p.ResolveIP(address(i)) + } + wg.Done() + }() + wg.Wait() +} diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 2ba08864ae8..96f036dd15d 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -14,6 +14,8 @@ - Thread safe sorted set {pull}21290[21290] - Copy Action store on upgrade {pull}21298[21298] - Include inputs in action store actions {pull}21298[21298] +- Fix issue where inputs without processors defined would panic {pull}21628[21628] +- Partial extracted beat result in failure to spawn beat {issue}21718[21718] ==== New features @@ -28,3 +30,9 @@ - Add support for EQL based condition on inputs {pull}20994[20994] - Send `fleet.host.id` to Endpoint Security {pull}21042[21042] - Add `install` and `uninstall` subcommands {pull}21206[21206] +- Use new form of fleet API paths {pull}21478[21478] +- Add `kubernetes` composable dynamic provider. {pull}21480[21480] +- Send updating state {pull}21461[21461] +- Add `elastic.agent.id` and `elastic.agent.version` to published events from filebeat and metricbeat {pull}21543[21543] +- Add `upgrade` subcommand to perform upgrade of installed Elastic Agent {pull}21425[21425] +- Update `fleet.yml` and Kibana hosts when a policy change updates the Kibana hosts {pull}21599[21599] diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index d2b3c117f67..3a4c54c414d 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -45,3 +45,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/elastic-agent for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/elastic-agent/pkg/agent/application/application.go b/x-pack/elastic-agent/pkg/agent/application/application.go index e003eed61a6..d721a8aa148 100644 --- a/x-pack/elastic-agent/pkg/agent/application/application.go +++ b/x-pack/elastic-agent/pkg/agent/application/application.go @@ -8,6 +8,7 @@ import ( "context" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/upgrade" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/warn" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" @@ -25,8 +26,12 @@ type reexecManager interface { ReExec(argOverrides ...string) } +type upgraderControl interface { + SetUpgrader(upgrader *upgrade.Upgrader) +} + // New creates a new Agent and bootstrap the required subsystem. -func New(log *logger.Logger, pathConfigFile string, reexec reexecManager) (Application, error) { +func New(log *logger.Logger, pathConfigFile string, reexec reexecManager, uc upgraderControl) (Application, error) { // Load configuration from disk to understand in which mode of operation // we must start the elastic-agent, the mode of operation cannot be changed without restarting the // elastic-agent. @@ -39,7 +44,7 @@ func New(log *logger.Logger, pathConfigFile string, reexec reexecManager) (Appli return nil, err } - return createApplication(log, pathConfigFile, rawConfig, reexec) + return createApplication(log, pathConfigFile, rawConfig, reexec, uc) } func createApplication( @@ -47,6 +52,7 @@ func createApplication( pathConfigFile string, rawConfig *config.Config, reexec reexecManager, + uc upgraderControl, ) (Application, error) { warn.LogNotGA(log) log.Info("Detecting execution mode") @@ -59,7 +65,7 @@ func createApplication( if isStandalone(cfg.Fleet) { log.Info("Agent is managed locally") - return newLocal(ctx, log, pathConfigFile, rawConfig) + return newLocal(ctx, log, pathConfigFile, rawConfig, reexec, uc) } log.Info("Agent is managed by Fleet") diff --git a/x-pack/elastic-agent/pkg/agent/application/emitter.go b/x-pack/elastic-agent/pkg/agent/application/emitter.go index d8a19492e2b..07f8e1f460b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/emitter.go +++ b/x-pack/elastic-agent/pkg/agent/application/emitter.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" @@ -18,7 +19,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" ) -type decoratorFunc = func(string, *transpiler.AST, []program.Program) ([]program.Program, error) +type decoratorFunc = func(*info.AgentInfo, string, *transpiler.AST, []program.Program) ([]program.Program, error) type filterFunc = func(*logger.Logger, *transpiler.AST) error type reloadable interface { @@ -36,16 +37,18 @@ type programsDispatcher interface { type emitterController struct { logger *logger.Logger + agentInfo *info.AgentInfo controller composable.Controller router programsDispatcher modifiers *configModifiers reloadables []reloadable // state - lock sync.RWMutex - config *config.Config - ast *transpiler.AST - vars []*transpiler.Vars + lock sync.RWMutex + updateLock sync.Mutex + config *config.Config + ast *transpiler.AST + vars []*transpiler.Vars } func (e *emitterController) Update(c *config.Config) error { @@ -91,6 +94,10 @@ func (e *emitterController) Set(vars []*transpiler.Vars) { } func (e *emitterController) update() error { + // locking whole update because it can be called concurrently via Set and Update method + e.updateLock.Lock() + defer e.updateLock.Unlock() + e.lock.RLock() cfg := e.config rawAst := e.ast @@ -112,14 +119,14 @@ func (e *emitterController) update() error { e.logger.Debug("Converting single configuration into specific programs configuration") - programsToRun, err := program.Programs(ast) + programsToRun, err := program.Programs(e.agentInfo, ast) if err != nil { return err } for _, decorator := range e.modifiers.Decorators { for outputType, ptr := range programsToRun { - programsToRun[outputType], err = decorator(outputType, ast, ptr) + programsToRun[outputType], err = decorator(e.agentInfo, outputType, ast, ptr) if err != nil { return err } @@ -135,12 +142,13 @@ func (e *emitterController) update() error { return e.router.Dispatch(ast.HashStr(), programsToRun) } -func emitter(ctx context.Context, log *logger.Logger, controller composable.Controller, router programsDispatcher, modifiers *configModifiers, reloadables ...reloadable) (emitterFunc, error) { +func emitter(ctx context.Context, log *logger.Logger, agentInfo *info.AgentInfo, controller composable.Controller, router programsDispatcher, modifiers *configModifiers, reloadables ...reloadable) (emitterFunc, error) { log.Debugf("Supported programs: %s", strings.Join(program.KnownProgramNames(), ", ")) init, _ := transpiler.NewVars(map[string]interface{}{}) ctrl := &emitterController{ logger: log, + agentInfo: agentInfo, controller: controller, router: router, modifiers: modifiers, @@ -211,17 +219,20 @@ func promoteProcessors(dict *transpiler.Dict) *transpiler.Dict { if p == nil { return dict } + var currentList *transpiler.List current, ok := dict.Find("processors") - currentList, isList := current.Value().(*transpiler.List) - if !isList { - return dict + if ok { + currentList, ok = current.Value().(*transpiler.List) + if !ok { + return dict + } } ast, _ := transpiler.NewAST(map[string]interface{}{ "processors": p, }) procs, _ := transpiler.Lookup(ast, "processors") nodes := nodesFromList(procs.Value().(*transpiler.List)) - if ok { + if ok && currentList != nil { nodes = append(nodes, nodesFromList(currentList)...) } dictNodes := dict.Value().([]transpiler.Node) diff --git a/x-pack/elastic-agent/pkg/agent/application/emitter_test.go b/x-pack/elastic-agent/pkg/agent/application/emitter_test.go index 32770eaa5df..b2286552f9f 100644 --- a/x-pack/elastic-agent/pkg/agent/application/emitter_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/emitter_test.go @@ -479,6 +479,181 @@ func TestRenderInputs(t *testing.T) { }), }, }, + "inputs without processors and vars with processors": { + input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/${var1.name}.log"), + })), + }), + })), + }), + })), + expected: transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/value1.log"), + })), + }), + })), + transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("custom", transpiler.NewStrVal("value1")), + })), + transpiler.NewKey("to", transpiler.NewStrVal("dynamic")), + })), + }), + })), + }), + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/value2.log"), + })), + }), + })), + transpiler.NewKey("processors", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("custom", transpiler.NewStrVal("value2")), + })), + transpiler.NewKey("to", transpiler.NewStrVal("dynamic")), + })), + }), + })), + }), + }), + varsArray: []*transpiler.Vars{ + mustMakeVarsP(map[string]interface{}{ + "var1": map[string]interface{}{ + "name": "value1", + }, + }, + "var1", + []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "custom": "value1", + }, + "to": "dynamic", + }, + }, + }), + mustMakeVarsP(map[string]interface{}{ + "var1": map[string]interface{}{ + "name": "value2", + }, + }, + "var1", + []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "custom": "value2", + }, + "to": "dynamic", + }, + }, + }), + }, + }, + "processors incorrectly a map": { + input: transpiler.NewKey("inputs", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/${var1.name}.log"), + })), + }), + })), + transpiler.NewKey("processors", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("invalid", transpiler.NewStrVal("value")), + })), + })), + }), + })), + expected: transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/value1.log"), + })), + }), + })), + transpiler.NewKey("processors", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("invalid", transpiler.NewStrVal("value")), + })), + })), + }), + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("type", transpiler.NewStrVal("logfile")), + transpiler.NewKey("streams", transpiler.NewList([]transpiler.Node{ + transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("paths", transpiler.NewList([]transpiler.Node{ + transpiler.NewStrVal("/var/log/value2.log"), + })), + }), + })), + transpiler.NewKey("processors", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("add_fields", transpiler.NewDict([]transpiler.Node{ + transpiler.NewKey("invalid", transpiler.NewStrVal("value")), + })), + })), + }), + }), + varsArray: []*transpiler.Vars{ + mustMakeVarsP(map[string]interface{}{ + "var1": map[string]interface{}{ + "name": "value1", + }, + }, + "var1", + []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "custom": "value1", + }, + "to": "dynamic", + }, + }, + }), + mustMakeVarsP(map[string]interface{}{ + "var1": map[string]interface{}{ + "name": "value2", + }, + }, + "var1", + []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": map[string]interface{}{ + "custom": "value2", + }, + "to": "dynamic", + }, + }, + }), + }, + }, } for name, test := range testcases { diff --git a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go index beab1b253d6..080b5efcb69 100644 --- a/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/enroll_cmd_test.go @@ -49,7 +49,7 @@ func TestEnroll(t *testing.T) { t.Run("fail to save is propagated", withTLSServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(` { @@ -102,7 +102,7 @@ func TestEnroll(t *testing.T) { t.Run("successfully enroll with TLS and save access api key in the store", withTLSServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(` { @@ -163,7 +163,7 @@ func TestEnroll(t *testing.T) { t.Run("successfully enroll when a slash is defined at the end of host", withServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(` { @@ -223,7 +223,7 @@ func TestEnroll(t *testing.T) { t.Run("successfully enroll without TLS and save access api key in the store", withServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte(` { @@ -283,7 +283,7 @@ func TestEnroll(t *testing.T) { t.Run("fail to enroll without TLS", withServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Write([]byte(` { diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go b/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go index ba324f2d7de..4544fa8a772 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_acker.go @@ -39,6 +39,10 @@ func newActionAcker( }, nil } +func (f *actionAcker) SetClient(client clienter) { + f.client = client +} + func (f *actionAcker) Ack(ctx context.Context, action fleetapi.Action) error { // checkin agentID := f.agentInfo.AgentID() diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go index 4bb9d2e6280..21e7bfd4589 100644 --- a/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go +++ b/x-pack/elastic-agent/pkg/agent/application/fleet_gateway.go @@ -253,3 +253,7 @@ func (f *fleetGateway) stop() { close(f.done) f.wg.Wait() } + +func (f *fleetGateway) SetClient(client clienter) { + f.client = client +} diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go index 81dc1444816..8821700ee74 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change.go @@ -5,17 +5,36 @@ package application import ( + "bytes" "context" "fmt" + "io" + "sort" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + + "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana" ) +type clientSetter interface { + SetClient(clienter) +} + type handlerPolicyChange struct { - log *logger.Logger - emitter emitterFunc + log *logger.Logger + emitter emitterFunc + agentInfo *info.AgentInfo + config *configuration.Configuration + store storage.Store + setters []clientSetter } func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetAcker) error { @@ -31,9 +50,93 @@ func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetA } h.log.Debugf("handlerPolicyChange: emit configuration for action %+v", a) + err = h.handleKibanaHosts(c) + if err != nil { + return err + } if err := h.emitter(c); err != nil { return err } return acker.Ack(ctx, action) } + +func (h *handlerPolicyChange) handleKibanaHosts(c *config.Config) (err error) { + cfg, err := configuration.NewFromConfig(c) + if err != nil { + return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig) + } + if kibanaEqual(h.config.Fleet.Kibana, cfg.Fleet.Kibana) { + // already the same hosts + return nil + } + + // only set protocol/hosts as that is all Fleet currently sends + prevProtocol := h.config.Fleet.Kibana.Protocol + prevHosts := h.config.Fleet.Kibana.Hosts + h.config.Fleet.Kibana.Protocol = cfg.Fleet.Kibana.Protocol + h.config.Fleet.Kibana.Hosts = cfg.Fleet.Kibana.Hosts + + // rollback on failure + defer func() { + if err != nil { + h.config.Fleet.Kibana.Protocol = prevProtocol + h.config.Fleet.Kibana.Hosts = prevHosts + } + }() + + client, err := fleetapi.NewAuthWithConfig(h.log, h.config.Fleet.AccessAPIKey, h.config.Fleet.Kibana) + if err != nil { + return errors.New( + err, "fail to create API client with updated hosts", + errors.TypeNetwork, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + } + reader, err := fleetToReader(h.agentInfo, h.config) + if err != nil { + return errors.New( + err, "fail to persist updated API client hosts", + errors.TypeUnexpected, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + } + err = h.store.Save(reader) + if err != nil { + return errors.New( + err, "fail to persist updated API client hosts", + errors.TypeFilesystem, errors.M("hosts", h.config.Fleet.Kibana.Hosts)) + } + for _, setter := range h.setters { + setter.SetClient(client) + } + return nil +} + +func kibanaEqual(k1 *kibana.Config, k2 *kibana.Config) bool { + if k1.Protocol != k2.Protocol { + return false + } + + sort.Strings(k1.Hosts) + sort.Strings(k2.Hosts) + if len(k1.Hosts) != len(k2.Hosts) { + return false + } + for i, v := range k1.Hosts { + if v != k2.Hosts[i] { + return false + } + } + return true +} + +func fleetToReader(agentInfo *info.AgentInfo, cfg *configuration.Configuration) (io.Reader, error) { + configToStore := map[string]interface{}{ + "fleet": cfg.Fleet, + "agent": map[string]interface{}{ + "id": agentInfo.AgentID(), + }, + } + data, err := yaml.Marshal(configToStore) + if err != nil { + return nil, err + } + return bytes.NewReader(data), nil +} diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go index ce4802b68e6..c30f886b0d1 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_policy_change_test.go @@ -9,6 +9,10 @@ import ( "sync" "testing" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" + "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -31,6 +35,8 @@ func (m *mockEmitter) Emitter(policy *config.Config) error { func TestPolicyChange(t *testing.T) { log, _ := logger.New("") ack := newNoopAcker() + agentInfo, _ := info.NewAgentInfo() + nullStore := &storage.NullStore{} t.Run("Receive a config change and successfully emits a raw configuration", func(t *testing.T) { emitter := &mockEmitter{} @@ -42,7 +48,14 @@ func TestPolicyChange(t *testing.T) { Policy: conf, } - handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} + cfg := configuration.DefaultConfiguration() + handler := &handlerPolicyChange{ + log: log, + emitter: emitter.Emitter, + agentInfo: agentInfo, + config: cfg, + store: nullStore, + } err := handler.Handle(context.Background(), action, ack) require.NoError(t, err) @@ -60,7 +73,14 @@ func TestPolicyChange(t *testing.T) { Policy: conf, } - handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} + cfg := configuration.DefaultConfiguration() + handler := &handlerPolicyChange{ + log: log, + emitter: emitter.Emitter, + agentInfo: agentInfo, + config: cfg, + store: nullStore, + } err := handler.Handle(context.Background(), action, ack) require.Error(t, err) @@ -69,6 +89,9 @@ func TestPolicyChange(t *testing.T) { func TestPolicyAcked(t *testing.T) { log, _ := logger.New("") + agentInfo, _ := info.NewAgentInfo() + nullStore := &storage.NullStore{} + t.Run("Config change should not ACK on error", func(t *testing.T) { tacker := &testAcker{} @@ -83,7 +106,14 @@ func TestPolicyAcked(t *testing.T) { Policy: config, } - handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} + cfg := configuration.DefaultConfiguration() + handler := &handlerPolicyChange{ + log: log, + emitter: emitter.Emitter, + agentInfo: agentInfo, + config: cfg, + store: nullStore, + } err := handler.Handle(context.Background(), action, tacker) require.Error(t, err) @@ -105,7 +135,14 @@ func TestPolicyAcked(t *testing.T) { Policy: config, } - handler := &handlerPolicyChange{log: log, emitter: emitter.Emitter} + cfg := configuration.DefaultConfiguration() + handler := &handlerPolicyChange{ + log: log, + emitter: emitter.Emitter, + agentInfo: agentInfo, + config: cfg, + store: nullStore, + } err := handler.Handle(context.Background(), action, tacker) require.NoError(t, err) diff --git a/x-pack/elastic-agent/pkg/agent/application/handler_action_upgrade.go b/x-pack/elastic-agent/pkg/agent/application/handler_action_upgrade.go index 4d0026d4d79..a4940cfe55b 100644 --- a/x-pack/elastic-agent/pkg/agent/application/handler_action_upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/handler_action_upgrade.go @@ -27,5 +27,21 @@ func (h *handlerUpgrade) Handle(ctx context.Context, a action, acker fleetAcker) return fmt.Errorf("invalid type, expected ActionUpgrade and received %T", a) } - return h.upgrader.Upgrade(ctx, action) + return h.upgrader.Upgrade(ctx, &upgradeAction{action}, true) +} + +type upgradeAction struct { + *fleetapi.ActionUpgrade +} + +func (a *upgradeAction) Version() string { + return a.ActionUpgrade.Version +} + +func (a *upgradeAction) SourceURI() string { + return a.ActionUpgrade.SourceURI +} + +func (a *upgradeAction) FleetAction() *fleetapi.ActionUpgrade { + return a.ActionUpgrade } diff --git a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go index e990b83bd49..b0abbe19e64 100644 --- a/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go +++ b/x-pack/elastic-agent/pkg/agent/application/info/agent_info.go @@ -4,6 +4,8 @@ package info +import "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" + // AgentInfo is a collection of information about agent. type AgentInfo struct { agentID string @@ -44,3 +46,13 @@ func ForceNewAgentInfo() (*AgentInfo, error) { func (i *AgentInfo) AgentID() string { return i.agentID } + +// Version returns the version for this Agent. +func (*AgentInfo) Version() string { + return release.Version() +} + +// Snapshot returns if this version is a snapshot. +func (*AgentInfo) Snapshot() bool { + return release.Snapshot() +} diff --git a/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go index 8f648887d10..bb319ce1569 100644 --- a/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go +++ b/x-pack/elastic-agent/pkg/agent/application/inspect_output_cmd.go @@ -8,11 +8,12 @@ import ( "context" "fmt" - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" - "github.com/elastic/beats/v7/libbeat/logp" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" @@ -37,14 +38,19 @@ func NewInspectOutputCmd(configPath, output, program string) (*InspectOutputCmd, // Execute tries to enroll the agent into Fleet. func (c *InspectOutputCmd) Execute() error { + agentInfo, err := info.NewAgentInfo() + if err != nil { + return err + } + if c.output == "" { - return c.inspectOutputs() + return c.inspectOutputs(agentInfo) } - return c.inspectOutput() + return c.inspectOutput(agentInfo) } -func (c *InspectOutputCmd) inspectOutputs() error { +func (c *InspectOutputCmd) inspectOutputs(agentInfo *info.AgentInfo) error { rawConfig, err := loadConfig(c.cfgPath) if err != nil { return err @@ -61,7 +67,7 @@ func (c *InspectOutputCmd) inspectOutputs() error { } if isStandalone(cfg.Fleet) { - return listOutputsFromConfig(l, rawConfig) + return listOutputsFromConfig(l, agentInfo, rawConfig) } fleetConfig, err := loadFleetConfig(rawConfig) @@ -71,11 +77,11 @@ func (c *InspectOutputCmd) inspectOutputs() error { return fmt.Errorf("no fleet config retrieved yet") } - return listOutputsFromMap(l, fleetConfig) + return listOutputsFromMap(l, agentInfo, fleetConfig) } -func listOutputsFromConfig(log *logger.Logger, cfg *config.Config) error { - programsGroup, err := getProgramsFromConfig(log, cfg) +func listOutputsFromConfig(log *logger.Logger, agentInfo *info.AgentInfo, cfg *config.Config) error { + programsGroup, err := getProgramsFromConfig(log, agentInfo, cfg) if err != nil { return err @@ -88,16 +94,16 @@ func listOutputsFromConfig(log *logger.Logger, cfg *config.Config) error { return nil } -func listOutputsFromMap(log *logger.Logger, cfg map[string]interface{}) error { +func listOutputsFromMap(log *logger.Logger, agentInfo *info.AgentInfo, cfg map[string]interface{}) error { c, err := config.NewConfigFrom(cfg) if err != nil { return err } - return listOutputsFromConfig(log, c) + return listOutputsFromConfig(log, agentInfo, c) } -func (c *InspectOutputCmd) inspectOutput() error { +func (c *InspectOutputCmd) inspectOutput(agentInfo *info.AgentInfo) error { rawConfig, err := loadConfig(c.cfgPath) if err != nil { return err @@ -114,7 +120,7 @@ func (c *InspectOutputCmd) inspectOutput() error { } if isStandalone(cfg.Fleet) { - return printOutputFromConfig(l, c.output, c.program, rawConfig) + return printOutputFromConfig(l, agentInfo, c.output, c.program, rawConfig) } fleetConfig, err := loadFleetConfig(rawConfig) @@ -124,11 +130,11 @@ func (c *InspectOutputCmd) inspectOutput() error { return fmt.Errorf("no fleet config retrieved yet") } - return printOutputFromMap(l, c.output, c.program, fleetConfig) + return printOutputFromMap(l, agentInfo, c.output, c.program, fleetConfig) } -func printOutputFromConfig(log *logger.Logger, output, programName string, cfg *config.Config) error { - programsGroup, err := getProgramsFromConfig(log, cfg) +func printOutputFromConfig(log *logger.Logger, agentInfo *info.AgentInfo, output, programName string, cfg *config.Config) error { + programsGroup, err := getProgramsFromConfig(log, agentInfo, cfg) if err != nil { return err @@ -164,16 +170,16 @@ func printOutputFromConfig(log *logger.Logger, output, programName string, cfg * } -func printOutputFromMap(log *logger.Logger, output, programName string, cfg map[string]interface{}) error { +func printOutputFromMap(log *logger.Logger, agentInfo *info.AgentInfo, output, programName string, cfg map[string]interface{}) error { c, err := config.NewConfigFrom(cfg) if err != nil { return err } - return printOutputFromConfig(log, output, programName, c) + return printOutputFromConfig(log, agentInfo, output, programName, c) } -func getProgramsFromConfig(log *logger.Logger, cfg *config.Config) (map[string][]program.Program, error) { +func getProgramsFromConfig(log *logger.Logger, agentInfo *info.AgentInfo, cfg *config.Config) (map[string][]program.Program, error) { monitor := noop.NewMonitor() router := &inmemRouter{} ctx, cancel := context.WithCancel(context.Background()) @@ -186,6 +192,7 @@ func getProgramsFromConfig(log *logger.Logger, cfg *config.Config) (map[string][ emit, err := emitter( ctx, log, + agentInfo, composableWaiter, router, &configModifiers{ diff --git a/x-pack/elastic-agent/pkg/agent/application/local_mode.go b/x-pack/elastic-agent/pkg/agent/application/local_mode.go index 5559089404e..b58e260cab6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/local_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/local_mode.go @@ -9,6 +9,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/upgrade" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" @@ -60,6 +61,8 @@ func newLocal( log *logger.Logger, pathConfigFile string, rawConfig *config.Config, + reexec reexecManager, + uc upgraderControl, ) (*Local, error) { cfg, err := configuration.NewFromConfig(rawConfig) if err != nil { @@ -112,6 +115,7 @@ func newLocal( emit, err := emitter( localApplication.bgContext, log, + agentInfo, composableCtrl, router, &configModifiers{ @@ -135,6 +139,17 @@ func newLocal( localApplication.source = cfgSource + // create a upgrader to use in local mode + upgrader := upgrade.NewUpgrader( + agentInfo, + cfg.Settings.DownloadConfig, + log, + []context.CancelFunc{localApplication.cancelCtxFn}, + reexec, + newNoopAcker(), + reporter) + uc.SetUpgrader(upgrader) + return localApplication, nil } diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go index 12a9c242780..e38685741c3 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode.go @@ -168,6 +168,7 @@ func newManaged( emit, err := emitter( managedApplication.bgContext, log, + agentInfo, composableCtrl, router, &configModifiers{ @@ -200,18 +201,25 @@ func newManaged( } managedApplication.upgrader = upgrade.NewUpgrader( + agentInfo, cfg.Settings.DownloadConfig, log, []context.CancelFunc{managedApplication.cancelCtxFn}, reexec, - acker) + acker, + combinedReporter) + policyChanger := &handlerPolicyChange{ + log: log, + emitter: emit, + agentInfo: agentInfo, + config: cfg, + store: store, + setters: []clientSetter{acker}, + } actionDispatcher.MustRegister( &fleetapi.ActionPolicyChange{}, - &handlerPolicyChange{ - log: log, - emitter: emit, - }, + policyChanger, ) actionDispatcher.MustRegister( @@ -261,6 +269,9 @@ func newManaged( if err != nil { return nil, err } + // add the gateway to setters, so the gateway can be updated + // when the hosts for Kibana are updated by the policy. + policyChanger.setters = append(policyChanger.setters, gateway) managedApplication.gateway = gateway return managedApplication, nil diff --git a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go index 81f2419f936..4325c5ce7a6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/managed_mode_test.go @@ -9,10 +9,14 @@ import ( "encoding/json" "testing" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configuration" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" @@ -32,18 +36,24 @@ func TestManagedModeRouting(t *testing.T) { log, _ := logger.New("") router, _ := newRouter(log, streamFn) + agentInfo, _ := info.NewAgentInfo() + nullStore := &storage.NullStore{} composableCtrl, _ := composable.New(log, nil) - emit, err := emitter(ctx, log, composableCtrl, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}}) + emit, err := emitter(ctx, log, agentInfo, composableCtrl, router, &configModifiers{Decorators: []decoratorFunc{injectMonitoring}}) require.NoError(t, err) actionDispatcher, err := newActionDispatcher(ctx, log, &handlerDefault{log: log}) require.NoError(t, err) + cfg := configuration.DefaultConfiguration() actionDispatcher.MustRegister( &fleetapi.ActionPolicyChange{}, &handlerPolicyChange{ - log: log, - emitter: emit, + log: log, + emitter: emit, + agentInfo: agentInfo, + config: cfg, + store: nullStore, }, ) diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go index 2b04126381b..3fc49ef17d3 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator.go @@ -7,6 +7,7 @@ package application import ( "fmt" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" ) @@ -28,7 +29,7 @@ const ( defaultOutputName = "default" ) -func injectMonitoring(outputGroup string, rootAst *transpiler.AST, programsToRun []program.Program) ([]program.Program, error) { +func injectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *transpiler.AST, programsToRun []program.Program) ([]program.Program, error) { var err error monitoringProgram := program.Program{ Spec: program.Spec{ @@ -63,7 +64,7 @@ func injectMonitoring(outputGroup string, rootAst *transpiler.AST, programsToRun } ast := rootAst.Clone() - if err := getMonitoringRule(monitoringOutputName).Apply(ast); err != nil { + if err := getMonitoringRule(monitoringOutputName).Apply(agentInfo, ast); err != nil { return programsToRun, err } @@ -93,6 +94,7 @@ func getMonitoringRule(outputName string) *transpiler.RuleList { return transpiler.NewRuleList( transpiler.Copy(monitoringOutputSelector, outputKey), transpiler.Rename(fmt.Sprintf("%s.%s", outputsKey, outputName), elasticsearchKey), + transpiler.InjectAgentInfo(), transpiler.Filter(monitoringKey, programsKey, outputKey), ) } diff --git a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go index f50bb74d5e8..6a3be4100be 100644 --- a/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go +++ b/x-pack/elastic-agent/pkg/agent/application/monitoring_decorator_test.go @@ -7,17 +7,22 @@ package application import ( "testing" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" ) func TestMonitoringInjection(t *testing.T) { + agentInfo, err := info.NewAgentInfo() + if err != nil { + t.Fatal(err) + } ast, err := transpiler.NewAST(inputConfigMap) if err != nil { t.Fatal(err) } - programsToRun, err := program.Programs(ast) + programsToRun, err := program.Programs(agentInfo, ast) if err != nil { t.Fatal(err) } @@ -25,7 +30,7 @@ func TestMonitoringInjection(t *testing.T) { GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) - newPtr, err := injectMonitoring(group, ast, ptr) + newPtr, err := injectMonitoring(agentInfo, group, ast, ptr) if err != nil { t.Error(err) continue GROUPLOOP @@ -83,12 +88,16 @@ GROUPLOOP: } func TestMonitoringInjectionDefaults(t *testing.T) { + agentInfo, err := info.NewAgentInfo() + if err != nil { + t.Fatal(err) + } ast, err := transpiler.NewAST(inputConfigMapDefaults) if err != nil { t.Fatal(err) } - programsToRun, err := program.Programs(ast) + programsToRun, err := program.Programs(agentInfo, ast) if err != nil { t.Fatal(err) } @@ -96,7 +105,7 @@ func TestMonitoringInjectionDefaults(t *testing.T) { GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) - newPtr, err := injectMonitoring(group, ast, ptr) + newPtr, err := injectMonitoring(agentInfo, group, ast, ptr) if err != nil { t.Error(err) continue GROUPLOOP @@ -154,12 +163,16 @@ GROUPLOOP: } func TestMonitoringInjectionDisabled(t *testing.T) { + agentInfo, err := info.NewAgentInfo() + if err != nil { + t.Fatal(err) + } ast, err := transpiler.NewAST(inputConfigMapDisabled) if err != nil { t.Fatal(err) } - programsToRun, err := program.Programs(ast) + programsToRun, err := program.Programs(agentInfo, ast) if err != nil { t.Fatal(err) } @@ -167,7 +180,7 @@ func TestMonitoringInjectionDisabled(t *testing.T) { GROUPLOOP: for group, ptr := range programsToRun { programsCount := len(ptr) - newPtr, err := injectMonitoring(group, ast, ptr) + newPtr, err := injectMonitoring(agentInfo, group, ast, ptr) if err != nil { t.Error(err) continue GROUPLOOP diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go index 28e93949fbf..3aea96da0ab 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_download.go @@ -6,6 +6,7 @@ package upgrade import ( "context" + "strings" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" downloader "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/download/localremote" @@ -16,7 +17,13 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri // do not update source config settings := *u.settings if sourceURI != "" { - settings.SourceURI = sourceURI + if strings.HasPrefix(sourceURI, "file://") { + // update the DropPath so the fs.Downloader can download from this + // path instead of looking into the installed downloads directory + settings.DropPath = strings.TrimPrefix(sourceURI, "file://") + } else { + settings.SourceURI = sourceURI + } } allowEmptyPgp, pgp := release.PGP() @@ -31,7 +38,7 @@ func (u *Upgrader) downloadArtifact(ctx context.Context, version, sourceURI stri return "", errors.New(err, "failed upgrade of agent binary") } - matches, err := verifier.Verify(agentName, version) + matches, err := verifier.Verify(agentName, version, agentArtifactName, true) if err != nil { return "", errors.New(err, "failed verification of agent binary") } diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go index 53920e6ecff..8d03fec3ff6 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/step_mark.go @@ -37,7 +37,7 @@ type updateMarker struct { } // markUpgrade marks update happened so we can handle grace period -func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action *fleetapi.ActionUpgrade) error { +func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action Action) error { prevVersion := release.Version() prevHash := release.Commit() if len(prevHash) > hashLen { @@ -49,7 +49,7 @@ func (h *Upgrader) markUpgrade(ctx context.Context, hash string, action *fleetap UpdatedOn: time.Now(), PrevVersion: prevVersion, PrevHash: prevHash, - Action: action, + Action: action.FleetAction(), } markerBytes, err := yaml.Marshal(marker) diff --git a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go index cac36ef7922..1a21bc154a1 100644 --- a/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go +++ b/x-pack/elastic-agent/pkg/agent/application/upgrade/upgrade.go @@ -20,6 +20,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -33,14 +34,26 @@ const ( // Upgrader performs an upgrade type Upgrader struct { + agentInfo *info.AgentInfo settings *artifact.Config log *logger.Logger closers []context.CancelFunc reexec reexecManager acker acker + reporter stateReporter upgradeable bool } +// Action is the upgrade action state. +type Action interface { + // Version to upgrade to. + Version() string + // SourceURI for download. + SourceURI() string + // FleetAction is the action from fleet that started the action (optional). + FleetAction() *fleetapi.ActionUpgrade +} + type reexecManager interface { ReExec(argOverrides ...string) } @@ -50,15 +63,21 @@ type acker interface { Commit(ctx context.Context) error } +type stateReporter interface { + OnStateChange(id string, name string, s state.State) +} + // NewUpgrader creates an upgrader which is capable of performing upgrade operation -func NewUpgrader(settings *artifact.Config, log *logger.Logger, closers []context.CancelFunc, reexec reexecManager, a acker) *Upgrader { +func NewUpgrader(agentInfo *info.AgentInfo, settings *artifact.Config, log *logger.Logger, closers []context.CancelFunc, reexec reexecManager, a acker, r stateReporter) *Upgrader { return &Upgrader{ + agentInfo: agentInfo, settings: settings, log: log, closers: closers, reexec: reexec, acker: a, - upgradeable: getUpgradable(), + reporter: r, + upgradeable: getUpgradeable(), } } @@ -68,20 +87,31 @@ func (u *Upgrader) Upgradeable() bool { } // Upgrade upgrades running agent -func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error { +func (u *Upgrader) Upgrade(ctx context.Context, a Action, reexecNow bool) (err error) { + // report failed + defer func() { + if err != nil { + if action := a.FleetAction(); action != nil { + u.reportFailure(ctx, action, err) + } + } + }() + if !u.upgradeable { return fmt.Errorf( "cannot be upgraded; must be installed with install sub-command and " + "running under control of the systems supervisor") } - sourceURI, err := u.sourceURI(a.Version, a.SourceURI) - archivePath, err := u.downloadArtifact(ctx, a.Version, sourceURI) + u.reportUpdating(a.Version()) + + sourceURI, err := u.sourceURI(a.Version(), a.SourceURI()) + archivePath, err := u.downloadArtifact(ctx, a.Version(), sourceURI) if err != nil { return err } - newHash, err := u.unpack(ctx, a.Version, archivePath) + newHash, err := u.unpack(ctx, a.Version(), archivePath) if err != nil { return err } @@ -91,7 +121,12 @@ func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error } if strings.HasPrefix(release.Commit(), newHash) { - return errors.New("upgrading to same version") + // not an error + if action := a.FleetAction(); action != nil { + u.ackAction(ctx, action) + } + u.log.Warn("upgrading to same version") + return nil } if err := copyActionStore(newHash); err != nil { @@ -108,7 +143,9 @@ func (u *Upgrader) Upgrade(ctx context.Context, a *fleetapi.ActionUpgrade) error return err } - u.reexec.ReExec() + if reexecNow { + u.reexec.ReExec() + } return nil } @@ -132,11 +169,7 @@ func (u *Upgrader) Ack(ctx context.Context) error { return nil } - if err := u.acker.Ack(ctx, marker.Action); err != nil { - return err - } - - if err := u.acker.Commit(ctx); err != nil { + if err := u.ackAction(ctx, marker.Action); err != nil { return err } @@ -148,6 +181,7 @@ func (u *Upgrader) Ack(ctx context.Context) error { return ioutil.WriteFile(markerFile, markerBytes, 0600) } + func (u *Upgrader) sourceURI(version, retrievedURI string) (string, error) { if strings.HasSuffix(version, "-SNAPSHOT") && retrievedURI == "" { return "", errors.New("snapshot upgrade requires source uri", errors.TypeConfig) @@ -159,11 +193,55 @@ func (u *Upgrader) sourceURI(version, retrievedURI string) (string, error) { return u.settings.SourceURI, nil } +// ackAction is used for successful updates, it was either updated successfully or to the same version +// so we need to remove updating state and get prevent from receiving same update action again. +func (u *Upgrader) ackAction(ctx context.Context, action fleetapi.Action) error { + if err := u.acker.Ack(ctx, action); err != nil { + return err + } + + if err := u.acker.Commit(ctx); err != nil { + return err + } + + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Running}, + ) + + return nil +} + +// report failure is used when update process fails. action is acked so it won't be received again +// and state is changed to FAILED +func (u *Upgrader) reportFailure(ctx context.Context, action fleetapi.Action, err error) { + // ack action + u.acker.Ack(ctx, action) + + // report failure + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Failed, Message: err.Error()}, + ) +} + +// reportUpdating sets state of agent to updating. +func (u *Upgrader) reportUpdating(version string) { + // report failure + u.reporter.OnStateChange( + "", + agentName, + state.State{Status: state.Updating, Message: fmt.Sprintf("Update to version '%s' started", version)}, + ) +} + func rollbackInstall(hash string) { os.RemoveAll(filepath.Join(paths.Data(), fmt.Sprintf("%s-%s", agentName, hash))) } -func getUpgradable() bool { +func getUpgradeable() bool { // only upgradeable if running from Agent installer and running under the // control of the system supervisor (or built specifically with upgrading enabled) return release.Upgradeable() || (install.RunningInstalled() && install.RunningUnderSupervisor()) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/common.go b/x-pack/elastic-agent/pkg/agent/cmd/common.go index 8ca5700f3c6..39093be71b6 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/common.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/common.go @@ -68,6 +68,7 @@ func NewCommandWithArgs(args []string, streams *cli.IOStreams) *cobra.Command { cmd.AddCommand(run) cmd.AddCommand(newInstallCommandWithArgs(flags, args, streams)) cmd.AddCommand(newUninstallCommandWithArgs(flags, args, streams)) + cmd.AddCommand(newUpgradeCommandWithArgs(flags, args, streams)) cmd.AddCommand(newEnrollCommandWithArgs(flags, args, streams)) cmd.AddCommand(newInspectCommandWithArgs(flags, args, streams)) diff --git a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go index 6a604554136..a92766b9c07 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/enroll.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/enroll.go @@ -119,7 +119,9 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args if fromInstall { force = true } - if !force { + + // prompt only when it is not forced and is already enrolled + if !force && (cfg.Fleet != nil && cfg.Fleet.Enabled == true) { confirm, err := c.Confirm("This will replace your current settings. Do you want to continue?", true) if err != nil { return errors.New(err, "problem reading prompt response") diff --git a/x-pack/elastic-agent/pkg/agent/cmd/include.go b/x-pack/elastic-agent/pkg/agent/cmd/include.go index 87506b88415..b955c85418f 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/include.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/include.go @@ -10,6 +10,7 @@ import ( _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/docker" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/env" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/host" + _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/kubernetes" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/local" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/localdynamic" _ "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable/providers/path" diff --git a/x-pack/elastic-agent/pkg/agent/cmd/run.go b/x-pack/elastic-agent/pkg/agent/cmd/run.go index 77beeb6fe1a..84dd8bd8a9a 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/run.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/run.go @@ -95,13 +95,13 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se rex := reexec.NewManager(rexLogger, execPath) // start the control listener - control := server.New(logger.Named("control"), rex) + control := server.New(logger.Named("control"), rex, nil) if err := control.Start(); err != nil { return err } defer control.Stop() - app, err := application.New(logger, pathConfigFile, rex) + app, err := application.New(logger, pathConfigFile, rex, control) if err != nil { return err } diff --git a/x-pack/elastic-agent/pkg/agent/cmd/upgrade.go b/x-pack/elastic-agent/pkg/agent/cmd/upgrade.go new file mode 100644 index 00000000000..81a5c82b4ab --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/cmd/upgrade.go @@ -0,0 +1,56 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli" +) + +func newUpgradeCommandWithArgs(flags *globalFlags, _ []string, streams *cli.IOStreams) *cobra.Command { + cmd := &cobra.Command{ + Use: "upgrade ", + Short: "Upgrade the currently running Elastic Agent to the specified version", + Args: cobra.ExactArgs(1), + Run: func(c *cobra.Command, args []string) { + if err := upgradeCmd(streams, c, flags, args); err != nil { + fmt.Fprintf(streams.Err, "%v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().StringP("source-uri", "s", "", "Source URI to download the new version from") + + return cmd +} + +func upgradeCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags, args []string) error { + fmt.Fprintln(streams.Out, "The upgrade process of Elastic Agent is currently EXPERIMENTAL and should not be used in production") + + version := args[0] + sourceURI, _ := cmd.Flags().GetString("source-uri") + + c := client.New() + err := c.Connect(context.Background()) + if err != nil { + return errors.New(err, "Failed communicating to running daemon", errors.TypeNetwork, errors.M("socket", control.Address())) + } + defer c.Disconnect() + version, err = c.Upgrade(context.Background(), version, sourceURI) + if err != nil { + return errors.New(err, "Failed trigger upgrade of daemon") + } + fmt.Fprintf(streams.Out, "Upgrade triggered to version %s, Elastic Agent is currently restarting\n", version) + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/control/control_test.go b/x-pack/elastic-agent/pkg/agent/control/control_test.go index 9454179ae60..5c56aed4691 100644 --- a/x-pack/elastic-agent/pkg/agent/control/control_test.go +++ b/x-pack/elastic-agent/pkg/agent/control/control_test.go @@ -20,7 +20,7 @@ import ( ) func TestServerClient_Version(t *testing.T) { - srv := server.New(newErrorLogger(t), nil) + srv := server.New(newErrorLogger(t), nil, nil) err := srv.Start() require.NoError(t, err) defer srv.Stop() diff --git a/x-pack/elastic-agent/pkg/agent/control/server/server.go b/x-pack/elastic-agent/pkg/agent/control/server/server.go index faa7982c814..0ce970c9256 100644 --- a/x-pack/elastic-agent/pkg/agent/control/server/server.go +++ b/x-pack/elastic-agent/pkg/agent/control/server/server.go @@ -7,14 +7,17 @@ package server import ( "context" "net" - - "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" + "sync" + "time" "google.golang.org/grpc" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/upgrade" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release" ) @@ -22,18 +25,28 @@ import ( type Server struct { logger *logger.Logger rex reexec.ExecManager + up *upgrade.Upgrader listener net.Listener server *grpc.Server + lock sync.RWMutex } // New creates a new control protocol server. -func New(log *logger.Logger, rex reexec.ExecManager) *Server { +func New(log *logger.Logger, rex reexec.ExecManager, up *upgrade.Upgrader) *Server { return &Server{ logger: log, rex: rex, + up: up, } } +// SetUpgrader changes the upgrader. +func (s *Server) SetUpgrader(up *upgrade.Upgrader) { + s.lock.Lock() + defer s.lock.Unlock() + s.up = up +} + // Start starts the GRPC endpoint and accepts new connections. func (s *Server) Start() error { if s.server != nil { @@ -100,10 +113,48 @@ func (s *Server) Restart(_ context.Context, _ *proto.Empty) (*proto.RestartRespo // Upgrade performs the upgrade operation. func (s *Server) Upgrade(ctx context.Context, request *proto.UpgradeRequest) (*proto.UpgradeResponse, error) { - // not implemented + s.lock.RLock() + u := s.up + s.lock.RUnlock() + if u == nil { + // not running with upgrader (must be controlled by Fleet) + return &proto.UpgradeResponse{ + Status: proto.ActionStatus_FAILURE, + Error: "cannot be upgraded; perform upgrading using Fleet", + }, nil + } + err := u.Upgrade(ctx, &upgradeRequest{request}, false) + if err != nil { + return &proto.UpgradeResponse{ + Status: proto.ActionStatus_FAILURE, + Error: err.Error(), + }, nil + } + // perform the re-exec after a 1 second delay + // this ensures that the upgrade response over GRPC is returned + go func() { + <-time.After(time.Second) + s.rex.ReExec() + }() return &proto.UpgradeResponse{ - Status: proto.ActionStatus_FAILURE, - Version: "", - Error: "not implemented", + Status: proto.ActionStatus_SUCCESS, + Version: request.Version, }, nil } + +type upgradeRequest struct { + *proto.UpgradeRequest +} + +func (r *upgradeRequest) Version() string { + return r.GetVersion() +} + +func (r *upgradeRequest) SourceURI() string { + return r.GetSourceURI() +} + +func (r *upgradeRequest) FleetAction() *fleetapi.ActionUpgrade { + // upgrade request not from Fleet + return nil +} diff --git a/x-pack/elastic-agent/pkg/agent/operation/common_test.go b/x-pack/elastic-agent/pkg/agent/operation/common_test.go index cc17733c656..e9d40bece87 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/common_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/common_test.go @@ -143,7 +143,7 @@ var _ download.Downloader = &DummyDownloader{} type DummyVerifier struct{} -func (*DummyVerifier) Verify(p, v string) (bool, error) { +func (*DummyVerifier) Verify(p, v, _ string, _ bool) (bool, error) { return true, nil } diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index fe33de852d1..c4d895eb6ee 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -161,7 +161,7 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [ ProgramSpec: program.Spec{ Name: metricsProcessName, Cmd: metricsProcessName, - Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName), + Artifact: fmt.Sprintf("%s/%s", artifactPrefix, metricsProcessName), }, Meta: map[string]interface{}{ configrequest.MetaConfigKey: mbConfig, diff --git a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go index 289693ca373..7626d339e57 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go +++ b/x-pack/elastic-agent/pkg/agent/operation/operation_verify.go @@ -66,7 +66,7 @@ func (o *operationVerify) Run(_ context.Context, application Application) (err e } }() - isVerified, err := o.verifier.Verify(o.program.BinaryName(), o.program.Version()) + isVerified, err := o.verifier.Verify(o.program.BinaryName(), o.program.Version(), o.program.ArtifactName(), true) if err != nil { return errors.New(err, fmt.Sprintf("operation '%s' failed to verify %s.%s", o.Name(), o.program.BinaryName(), o.program.Version()), diff --git a/x-pack/elastic-agent/pkg/agent/program/program.go b/x-pack/elastic-agent/pkg/agent/program/program.go index 25b56081e68..f3f17d06b9d 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program.go +++ b/x-pack/elastic-agent/pkg/agent/program/program.go @@ -47,7 +47,7 @@ func (p *Program) Configuration() map[string]interface{} { // Programs take a Tree representation of the main configuration and apply all the different // programs rules and generate individual configuration from the rules. -func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { +func Programs(agentInfo transpiler.AgentInfo, singleConfig *transpiler.AST) (map[string][]Program, error) { grouped, err := groupByOutputs(singleConfig) if err != nil { return nil, errors.New(err, errors.TypeConfig, "fail to extract program configuration") @@ -55,7 +55,7 @@ func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { groupedPrograms := make(map[string][]Program) for k, config := range grouped { - programs, err := detectPrograms(config) + programs, err := detectPrograms(agentInfo, config) if err != nil { return nil, errors.New(err, errors.TypeConfig, "fail to generate program configuration") } @@ -65,11 +65,11 @@ func Programs(singleConfig *transpiler.AST) (map[string][]Program, error) { return groupedPrograms, nil } -func detectPrograms(singleConfig *transpiler.AST) ([]Program, error) { +func detectPrograms(agentInfo transpiler.AgentInfo, singleConfig *transpiler.AST) ([]Program, error) { programs := make([]Program, 0) for _, spec := range Supported { specificAST := singleConfig.Clone() - err := spec.Rules.Apply(specificAST) + err := spec.Rules.Apply(agentInfo, specificAST) if err != nil { return nil, err } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index c15510b6655..8c2cf8c499f 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -437,7 +437,7 @@ func TestConfiguration(t *testing.T) { ast, err := transpiler.NewAST(m) require.NoError(t, err) - programs, err := Programs(ast) + programs, err := Programs(&fakeAgentInfo{}, ast) if test.err { require.Error(t, err) return @@ -478,3 +478,17 @@ func TestConfiguration(t *testing.T) { }) } } + +type fakeAgentInfo struct{} + +func (*fakeAgentInfo) AgentID() string { + return "agent-id" +} + +func (*fakeAgentInfo) Version() string { + return "8.0.0" +} + +func (*fakeAgentInfo) Snapshot() bool { + return false +} diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go index 3b314bfa3f4..adc13938ae2 100644 --- a/x-pack/elastic-agent/pkg/agent/program/supported.go +++ b/x-pack/elastic-agent/pkg/agent/program/supported.go @@ -21,7 +21,7 @@ func init() { // spec/filebeat.yml // spec/heartbeat.yml // spec/metricbeat.yml - unpacked := packer.MustUnpack("eJzEWEtz47rR3X8/Y7b3q4QER06YqixEufiSRI9oGwCxIwCJpARSuuZDolL57ymQFB+yPXMnt2qymNIIhoDuRvfpc/pfX/LTlv11m/HTMcmKv9Sp+PKPLzQ1C/JyjHw0OzBLP9FsE70CuOfYPXH7sAyAenhKDEFT/0yBKPlCvRLkqSwVynZzilnmn0hq7vnjMSLDGQWxIFhknmAZOQXg9cF5DLSnx2gZgFgEoNiFaHbllpnTx+Ny9WyIrQX3GJATtV4fFsk8chbGOcD+8SmZJ+Nz2WBb0u2LWcqvT9ExchbzaPU8T3gK6xCRmdOtcUsUBOmqtHF9nS+ZpV+5Kc/zlABd8qfoWDgW/EqQtyOpyMnLcSl/59hGzK3owVm4H/v/7LT7LLMm2rqze144C7c/2xnZtXpWVWbxOkC+uFuvCfYqjt09wetkdM4n9072l9tUnD/y1dvPz4vMqAnUVZqKkml+TK3zwyJRIoJjEah6GqKLuMWOWaYSPh4jJ4UlsY0qRDNlhT0RaLAOsd/HM8Buxq5djG4xR7N3Pr+3xVWpBa9tvMlpa+pXbrsiQMqDYxf6olunti+Y0EGALirBt7gaV4IuItD8iu2PUYhmZ479a/e3N4IPD47tz5j12r0diakNxWCnMs7PZRODVOTcgjXW7vbanqAW3HNLr58S40QzQ+X2unvrQmxfmlyPg/QiyLzzNTVzjuAoDw2FZVA0Pt3Oa3LOr/p4A5gT5ClUc69PiUGJPA9vygB5e4K9KwbmOYS69C13LJITBJVVWpyC1CwDqExztP+7eQ43TU0VAZ7f1ZKRUgsK3tnMMpgP8Z0Xju0KinRA2jtv682/EMDZU2LEAfAE07xdgI0TBoXYbnp/a4LUiqdw1+ztfBzHLAQiCdAsnrzz/n3MJ2/WxqT/Pn33eeFYusptQ7351NiByYkBUdHouOQgFnR/jKgFS6L5x+XC/1t7pq8vn+e/OY/zKECzg2NdBE25Ei6iwxaIktlQYZpych6/RuuFEdN0E4WWeX0GcCbPoBpU5J7d8zlyAcwD7Ckh8q4EmXUAomy5Of7zy/+3kLtLxJZuw3eQK6EGuSLAmxvMNuUYpDDm81MLa4lBnUQ1neQcOZknuA3Pq1Tk9HkmaGom1IKHb0imryeaPfd7M19QbOQB9sUqhWWA3JygjU5SM2fgNVkt5snqtf2kyCwDxAVFsOSLWUGBL77hqGCWuQ9rtU2dhZM7C6fwn+WnW8jnJAAWRELF6Hxuuyp5nuzNKeBZiGbZKr0InsL8G/JFkMHMEcoywK4SIhIH2ubBsWRM/OuqaQcwIchUfggdSZMav8tywkCUxIJfbynIbXGW8aaWnrFzUxonmp4klOyY5tcEmQXWjJq2qV31KWnpJQZeRVOSh8hTWiiQLc3fBYgoBHfw38LOg2NdKqKtG2ihyDzfw+odZNUcXSbwFAD9vIV6TK3Ljlv6jlriyh8HmHUWhkKvx+hmMzuPS+ydrSUF+nlcwgTHe4INpcmpzFNYCmOK183bh2jTfPaw1ryze2ap3kCRhCj5Tne2KlTV8xB7yrTchUKadxnFNFv/t34MMU9hSjW3g1TZGps66t6K1BQoD47Vle751oL+Pqxpvc/LrgUqTFIXs/UBA2m3ev3s3e7tDbEv6Mt7PyZ3nj+F4mlbsfv8HtpHapYMXGLeU6L5xK4mrzfj2Kkxs40BUvv1S0U6atX8fxzvJi+IoNmm6qhPUyfj+5yFIeu15Av9yi3/JOGUaf4hRF/v7oGgwQHN3zNpn+WdPzlHJfb8wbHhgc2ntsi7V8CvAlBIPyJi6fsQwPrunJwCVrEUHkLs7Ri4VBxcKiJzqllbv/e/1q9b7MnfPTi2N5O/ucXhj7Qujj2BwQet5ge/I5apBLDHqr5+WAoLqhHRtNCXSY23VMbyY26ZPT6t0llMEbxKLCY/0XLv7i+b79iTFEDmpew3CsHu7p6+DNTEeVdTHRVQttgQXU7fUTXZlk2NAS+nGjxwYCoBiAbswCeVpa9Fm3f+kSNnhCuXiiM/pZqkoO5sOM+raObHIZoJNtTIgQLvrcdhSReAXhFwEStsqEHmqcGw98ht/4zBQG2Hs2OF28bvDOjlsFbEJC3i4fskXwqG/dHvZ4JbJKca6+2g1zXwkKkSSyhjekUsMVAXez36v6cQS5Sj7/c5qgTavD+fI/887IVliIf4ciDKpob/LJ22+n7+KaVu+nxLN/dUM/ocJJlbSWy8O7fBfTLiPWMa/iEtHt5vxI36tYpj/8zH9BDAGZM+pa+fUL/+7vJm2+75EH1L5mfHMkuyMI4B9lYEH46uXXTn+/pqMc8IusRM80+B5okAu/twwXJnwWuC/BOrWS59dEGbN24taaHkA56st6NbH5Y3Ohhvw7fiAz74bEEpp1u+k3oFkb1nstbKTMfMe87GAFQ4npchuhQ/4ne3vdyCBbOavlL2/f5RTQN0ud5xtjt+p1bEetW3C/UcIO9thVrJMeGdqRrT1MwIUmVPGZ/fyKHpXtmX+ImmrKRN7zjrxIIJRyzBd6OF5t3tdTWJRzbumU0efcW3Hv9yjLZag7cf1IC340AooanXBHGxtedDf73h/aS3GrV8H5x5M4lXBMscWFerJO9x+3O8/I7M/AHOflqHH8jNu3qc3DvsGdV8tp5wioYTj/nI4mN59Ufr7H9VW+m2eEvYB8X1gqDCUrHvxNWeIik0VMFt9xSAToS1c40I1X0BXAn2VbaYnail/KhYbnsliTxTy1TIj0TaXbFQpB/Ii/p1haXOzotO+35PpA3nY7/m6E7QWXpGpBiqZ3nTbB/VA0GuSmqXSzDhlkiDlmQ3BcVqvSDYr0PkdQVmVEzzJ7O6Nila0jGZlU3mR2pF7Ga2UJJFQ7ykICi3SO1nQ7IZyHgTvHmQQEOB3xTzKt1UTBNXCVKrTBR0MZMk7yZKlsPM4uOCH4u7EM0OBEe3htcQlqfEuPl4bRukKMO0mc10JEndMdutAgCvDOh98VAw2wVAL0l6ObUiVZQMwJqbekwyvyclvdjs8q0TALXMHYr62WbKUr14LwL8aljzbvZ0dqoxe7ybS34gbD4RE01zxsDMqfmJaGvvHu4cgcN732cV7clIS+y3lieYvWmaUy+I6qYuTp2Y7HO1HVBMxGGCN3e2an6FweXEtM10DnUTXaM3mgjIn/Kjf8OEINKA2S8Whu8IONb4iVvxjqUwIzjuhwgfkO62KSVf31agwzFtffguufu1hPBPil74eZP+ngi2XVnj2+WjvvnWDmZ+WyX56X2MukYq73g8Ru54dtwKtDJAqpiKqm7IMNk7EFuJ3xxdxCAM1DgEcBdgtw7u56ldjvQ4AaAysavJlZvNnhga8x8Rj6Pf/YxYvZtp/1qB23y/jme9v0ok34n7nxIzdMIrvov1k966Sj8aVPV982eE0bRnfz4XfyNY9lW9lvX5EWmb+NIOgJv6/DMkriFuWit8W+L2HRL37//7TwAAAP//S6KvFQ==") + unpacked := packer.MustUnpack("eJy8WF1zozoSfd+fMa93axdEnF226j4YUnzFJmOSSEJvSLIBW2DfALbx1v73LQHmw0lm7uxszcOUByKk7lb36XP631+Kw5r9fZ3zwz7Ny7/Vmfjyry80s0ryso8DNNsxWz/QfBW/Arjl2DtwZ/cYAnX3lBqCZsGJAlFxU70Q5KssE8p6dUhYHhxIZm35wz4mwx4lsSEwc1+wnBxC8HrvPoTa00P8GIJEhKDcRGh24bZV0If94+LZEGsbbjEgB2q/3pvpPHZN4xTiYP+UztPxvmywLe3WJSzjl6d4H7vmPF48z1OewTpCZOZ277gtSoJ0Vdq4vMwfma1fuCX385UQnYuneF+6NrwjyN+QTBTkZf8ov3MdI+F2fO+a3sf+P7vtOtuqibbs7J6Xrun1e7sjuxbPqspsXocoEDfva4L9I8feluBlOtrnk3Mn66t1Jk4f+epv5yczN2oCdZVmomJakFD7dG+mSkxwIkJVzyJ0FtfYMdtSood97GawIo5xjNBMWWBfhBqsIxz08Qyxl7NLF6NrzNHsnc/vbfFUasNLG29yWFv6hTueCJFy7zqlbnbvqRMIJnQQorNK8DWuxoWgswi14Mi2+zhCsxPHwaX72xvBu3vXCWbMfu3ujiTUgWKwUxnn52MTg0wU3IY11m7WOr6gNtxyW6+fUuNAc0PlzrK761KsX5pcT8LsLMi88zWzCo7gKA8NheVQND5d92tyLjj28QawIMhXqOZdnlKDErkfXlUh8rcE+xcMrFMEdelb4dqkIAgqi6w8hJlVhVCZ5mj/d+sUrZqaKkM8v6klI6M2FLyzmeWwGOI7L13HExTpgLRnXt83/yIAZ0+pkYTAF0zzNyE2DhiUYr3q/a0JUo88g5tmbefjOGYREGmIZsnknrfvYz65szYm/fP03uela+sqdwz16lNjByYHBsSRxvtHDhJBt/uY2rAiWrB/NIN/tHsG+uPz/Df3YR6HaLZz7bOgGVciM96tgaiYAxWmKQf34S5emkZCs1Uc2dblGcCZ3INqUJFrNs+n2AOwCLGvRMi/EGTVIYjzx9X+9y9/bSF3k4o1XUfvIFdCDfJEiFdXmG3KMcxgwueHFtZSg7qparnpKXZzX3AHnhaZKOjzTNDMSqkNd1+RTF9fNGtu1+aBoNgoQhyIRQarEHkFQSudZFbBwGu6MOfp4rX9pciqQsQFRbDi5qykIBBfcVwy29pGtdqmjukWrumWwbP89Up5nQTAkkioGO3PHU8lz5O1BQU8j9AsX2RnwTNYfEWBCHOYu0J5DLGnRIgkoba6d20Zk+CyaNoBTAmylO9CR9qkxh+ynDAQFbHh3TUFuSNOMt7U1nN2akrjQLODhJIN04KaIKvEmlHTNrWPfUraeoWBf6QZKSLkKy0UyJYWbEJEFII7+G9h5961z0eiLRtoocg63cLqDWTVHJ0n8BQC/bSGekLt84bb+oba4sIfBph1TUOhl318tZmdxiX2ztaKAv00LmGCky3BhtLkVO4rLIMJxcvm7iO0an57WGvu2TuxTG+gSEKUvKcbWxWq6kWEfWVa7kIhzb2MYpov/1c/hphnMKOa10GqbI1NHXV3RWoKlHvX7kr3dG1B/xzeab3Pj10LVJikLlbrAwbSbvXy2b3d2hvhQNCX935Mzjx9CsXTtuL0+T20j8yqGDgnvKdE84ldTV6vxrFTE+YYA6T2789H0lGr5v/jeDd5QQTNV8eO+jR1Mj7PNQ1ZrxU39Qu3g4OEU6YFuwjd3ZwDQYMDWrBl0j7bP32yj0qc+b3rwB2bT22RZy9AcAxBKf2Iia1vIwDrm30KCtiRZXAXYX/DwPnIwflIZE4175bv/a/1yxr78rt71/Fn8ptrHP5M6+LYFxh80Gq+8x2xLSWEPVb19cMyWFKNiKaFvkxqvKUydpBw2+rxaZHNEorgRWIx+YGWe3N+1TxjX1IAmZey3ygEe5tb+jJQE/ddTXVUQFljQ3Q5fUPVZFu2NAb8gmpwx4GlhCAesAMfVJa9lm3eBXuO3BGunI8cBRnVJAX1ZsN+/pHmQRKhmWBDjewo8N96HJZ0AehHAs5igQ01zH01HNbuuROcMBio7bB3onDH+IMBvRrelQnJymR4nuRLyXAw+n4muE0KqrHeDnpZAh9ZKrGFMqZXxBYDdXGWo//7CrFFNXq+zVEl1Ob9/hwFp2EtrCI8xJcDUTU1/LN02u77+aeUuunzq2lPbThEHkiJl0mcX2tKg7dNz/hTeD3q+z9EFQc+1d9DR9NG9yrWuKHcwsz5nqC7+yn1G85eZD9PAxfmPO8wKV80uMHfQkTewmdWuCaXnEjK4EtksoMZ/95TxmQdvZUfcMZnG0rJ3cYm80si+9PkXStFXavoeR0DUOF4XkXoXH6PA17XchuWzG56T9Vzggc1C9H5csPrbjigeiT2q7421VOI/LcFamXJhJtmakIzKydIlX1nvH8jmaZrZe/iB5qxijb95aQTG6YcsRTfjB8a3uwsj5N45OO+2uTIHb7ygJc2N9u8uK0Tf8OBUCJLrwniYu3Mhx587QmT/mvU8n5w7s8kphEcHEJteVykRY/tn2PqN6Tod7D401r9QJLe1Ozk3GHNCBfy5YR3NDU+5izmxxKsl2oAzpjcJ3v9UIZdMWfzvIu/pvOTa1sVMY19iP0Fwbu955RHjgO5Rpd1RNA5YZqMqy9C7G0js6mhmqDgwGpWSFs90OK2V8t6lHzcl/1u79W7x2ttZevyLWUfFNcLggrLxLYTYFuKpBhRBXe8Qwg6odbOPmJU9wVwIThQmTk7UFv5XrFc10qieaK2pZDvCbmbYqFI35EX9W6BpRYvyk4ff0vIDfvjoOboRvTZek6kYKpnRdOQH9QdQZ5Kao9LMOG2yMKWiDcFxWq9JDioI+R3BWYcmRZM5nltUrTEZDJPm8yY1CNxmvlDRcyGnEnRUK2R2s+PpFCQ8SZ4dS+BhoKgKeZFtjoyTVwkSC1yUVJzJongVbg8DnONjwt+3KwiNNsRHF+bYkNqnlLj6uOlbUCiirJmftMRKXXDHO8YAnhhQO+Lh4LZJgR6RbLzoRWyomIA1tzSE5IHPXHpBWmXb51IqGXuUNTPPzOW6eV7oRAch3f+1Z7OTjVhDzezyw/EzyeCY0s1Y4aBVVDrE2HXnj2cOQKH977PjrQnLC35X9u+YM6qaU69aKqbujh0grPP1XaIMRGQKV7d2KoFRwzOB6atprOqqzAb3dFEZP6QH/0dpgSRBsx+sXh8R9Kxxg/cTjYsgznBST9o+ICYt00pvXtbgA7HtOXumwTw15LGnxTG8PMm/S2h7HiyxtePD/rqazu8+W2RFof3MeoaqTzjYR974/lyK+KqEKliKrw6YjtZOww9JH5zdBYDIVWTCMBNiL06vJ25djnS4wToiewoV642+2JozH9GYI6++xFBezP3/rUiuHm+jOfBv0pI3wwA/s+CZ8ox5Le3uUVy7zjOhYajSM7xvjf0ffJHxNNk3+FuOwI2mr+PSN3HAmriS3XN958SUa1w6oned0XUf/7y3wAAAP//NCXBlQ==") SupportedMap = make(map[string]Spec) for f, v := range unpacked { diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml index 8edc27061b0..38b251d95dc 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_output_true-filebeat.yml @@ -16,6 +16,12 @@ filebeat: target: "event" fields: dataset: generic + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false output: elasticsearch: enabled: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml index 8bd5d93a3b9..6e768db6aa4 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/enabled_true-filebeat.yml @@ -17,6 +17,12 @@ filebeat: target: "event" fields: dataset: generic + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false output: elasticsearch: hosts: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml index b996e13b531..01ee955e4ec 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-filebeat.yml @@ -18,6 +18,12 @@ filebeat: target: "event" fields: dataset: generic + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false - type: log paths: - /var/log/hello3.log @@ -36,6 +42,12 @@ filebeat: target: "event" fields: dataset: generic + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false output: elasticsearch: hosts: diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml index c62882ff6da..d09e80accf1 100644 --- a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-metricbeat.yml @@ -15,6 +15,12 @@ metricbeat: target: "event" fields: dataset: docker.status + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false - module: docker metricsets: [info] index: metrics-generic-default @@ -30,6 +36,12 @@ metricbeat: target: "event" fields: dataset: generic + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false - module: apache metricsets: [info] index: metrics-generic-testing @@ -48,7 +60,12 @@ metricbeat: target: "event" fields: dataset: generic - + - add_fields: + target: "elastic" + fields: + agent.id: agent-id + agent.version: 8.0.0 + agent.snapshot: false output: elasticsearch: hosts: [127.0.0.1:9200, 127.0.0.1:9300] diff --git a/x-pack/elastic-agent/pkg/agent/storage/storage.go b/x-pack/elastic-agent/pkg/agent/storage/storage.go index b37c6e06ab3..2435311486a 100644 --- a/x-pack/elastic-agent/pkg/agent/storage/storage.go +++ b/x-pack/elastic-agent/pkg/agent/storage/storage.go @@ -20,7 +20,9 @@ import ( const perms os.FileMode = 0600 -type store interface { +// Store saves the io.Reader. +type Store interface { + // Save the io.Reader. Save(io.Reader) error } @@ -62,12 +64,12 @@ type ReplaceOnSuccessStore struct { target string replaceWith []byte - wrapped store + wrapped Store } // NewReplaceOnSuccessStore takes a target file and a replacement content and will replace the target // file content if the wrapped store execution is done without any error. -func NewReplaceOnSuccessStore(target string, replaceWith []byte, wrapped store) *ReplaceOnSuccessStore { +func NewReplaceOnSuccessStore(target string, replaceWith []byte, wrapped Store) *ReplaceOnSuccessStore { return &ReplaceOnSuccessStore{ target: target, replaceWith: replaceWith, diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go index 5ad790eb31e..29ff1786d1e 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules.go @@ -14,6 +14,13 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) +// AgentInfo is an interface to get the agent info. +type AgentInfo interface { + AgentID() string + Version() string + Snapshot() bool +} + // RuleList is a container that allow the same tree to be executed on multiple defined Rule. type RuleList struct { Rules []Rule @@ -21,15 +28,15 @@ type RuleList struct { // Rule defines a rule that can be Applied on the Tree. type Rule interface { - Apply(*AST) error + Apply(AgentInfo, *AST) error } // Apply applies a list of rules over the same tree and use the result of the previous execution // as the input of the next rule, will return early if any error is raise during the execution. -func (r *RuleList) Apply(ast *AST) error { +func (r *RuleList) Apply(agentInfo AgentInfo, ast *AST) error { var err error for _, rule := range r.Rules { - err = rule.Apply(ast) + err = rule.Apply(agentInfo, ast) if err != nil { return err } @@ -73,6 +80,8 @@ func (r *RuleList) MarshalYAML() (interface{}, error) { name = "inject_index" case *InjectStreamProcessorRule: name = "inject_stream_processor" + case *InjectAgentInfoRule: + name = "inject_agent_info" case *MakeArrayRule: name = "make_array" case *RemoveKeyRule: @@ -154,6 +163,8 @@ func (r *RuleList) UnmarshalYAML(unmarshal func(interface{}) error) error { r = &InjectIndexRule{} case "inject_stream_processor": r = &InjectStreamProcessorRule{} + case "inject_agent_info": + r = &InjectAgentInfoRule{} case "make_array": r = &MakeArrayRule{} case "remove_key": @@ -181,7 +192,7 @@ type SelectIntoRule struct { } // Apply applies select into rule. -func (r *SelectIntoRule) Apply(ast *AST) error { +func (r *SelectIntoRule) Apply(_ AgentInfo, ast *AST) error { target := &Dict{} for _, selector := range r.Selectors { @@ -214,7 +225,7 @@ type RemoveKeyRule struct { } // Apply applies remove key rule. -func (r *RemoveKeyRule) Apply(ast *AST) error { +func (r *RemoveKeyRule) Apply(_ AgentInfo, ast *AST) error { sourceMap, ok := ast.root.(*Dict) if !ok { return nil @@ -250,7 +261,7 @@ type MakeArrayRule struct { } // Apply applies make array rule. -func (r *MakeArrayRule) Apply(ast *AST) error { +func (r *MakeArrayRule) Apply(_ AgentInfo, ast *AST) error { sourceNode, found := Lookup(ast, r.Item) if !found { return nil @@ -286,7 +297,7 @@ type CopyToListRule struct { } // Apply copies specified node into every item of the list. -func (r *CopyToListRule) Apply(ast *AST) error { +func (r *CopyToListRule) Apply(_ AgentInfo, ast *AST) error { sourceNode, found := Lookup(ast, r.Item) if !found { // nothing to copy @@ -347,7 +358,7 @@ type CopyAllToListRule struct { } // Apply copies all nodes into every item of the list. -func (r *CopyAllToListRule) Apply(ast *AST) error { +func (r *CopyAllToListRule) Apply(agentInfo AgentInfo, ast *AST) error { // get list of nodes astMap, err := ast.Map() if err != nil { @@ -370,7 +381,7 @@ func (r *CopyAllToListRule) Apply(ast *AST) error { continue } - if err := CopyToList(item, r.To, r.OnConflict).Apply(ast); err != nil { + if err := CopyToList(item, r.To, r.OnConflict).Apply(agentInfo, ast); err != nil { return err } } @@ -393,7 +404,7 @@ type FixStreamRule struct { } // Apply stream fixes. -func (r *FixStreamRule) Apply(ast *AST) error { +func (r *FixStreamRule) Apply(_ AgentInfo, ast *AST) error { const defaultDataset = "generic" const defaultNamespace = "default" @@ -526,7 +537,7 @@ type InjectIndexRule struct { } // Apply injects index into input. -func (r *InjectIndexRule) Apply(ast *AST) error { +func (r *InjectIndexRule) Apply(_ AgentInfo, ast *AST) error { inputsNode, found := Lookup(ast, "inputs") if !found { return nil @@ -583,7 +594,7 @@ type InjectStreamProcessorRule struct { } // Apply injects processor into input. -func (r *InjectStreamProcessorRule) Apply(ast *AST) error { +func (r *InjectStreamProcessorRule) Apply(_ AgentInfo, ast *AST) error { inputsNode, found := Lookup(ast, "inputs") if !found { return nil @@ -665,6 +676,63 @@ func InjectStreamProcessor(onMerge, streamType string) *InjectStreamProcessorRul } } +// InjectAgentInfoRule injects agent information into each rule. +type InjectAgentInfoRule struct{} + +// Apply injects index into input. +func (r *InjectAgentInfoRule) Apply(agentInfo AgentInfo, ast *AST) error { + inputsNode, found := Lookup(ast, "inputs") + if !found { + return nil + } + + inputsList, ok := inputsNode.Value().(*List) + if !ok { + return nil + } + + for _, inputNode := range inputsList.value { + inputMap, ok := inputNode.(*Dict) + if !ok { + continue + } + + // get processors node + processorsNode, found := inputMap.Find("processors") + if !found { + processorsNode = &Key{ + name: "processors", + value: &List{value: make([]Node, 0)}, + } + + inputMap.value = append(inputMap.value, processorsNode) + } + + processorsList, ok := processorsNode.Value().(*List) + if !ok { + return errors.New("InjectAgentInfoRule: processors is not a list") + } + + // elastic.agent + processorMap := &Dict{value: make([]Node, 0)} + processorMap.value = append(processorMap.value, &Key{name: "target", value: &StrVal{value: "elastic"}}) + processorMap.value = append(processorMap.value, &Key{name: "fields", value: &Dict{value: []Node{ + &Key{name: "agent.id", value: &StrVal{value: agentInfo.AgentID()}}, + &Key{name: "agent.version", value: &StrVal{value: agentInfo.Version()}}, + &Key{name: "agent.snapshot", value: &BoolVal{value: agentInfo.Snapshot()}}, + }}}) + addFieldsMap := &Dict{value: []Node{&Key{"add_fields", processorMap}}} + processorsList.value = mergeStrategy("").InjectItem(processorsList.value, addFieldsMap) + } + + return nil +} + +// InjectAgentInfo creates a InjectAgentInfoRule +func InjectAgentInfo() *InjectAgentInfoRule { + return &InjectAgentInfoRule{} +} + // ExtractListItemRule extract items with specified name from a list of maps. // The result is store in a new array. // Example: @@ -679,7 +747,7 @@ type ExtractListItemRule struct { } // Apply extracts items from array. -func (r *ExtractListItemRule) Apply(ast *AST) error { +func (r *ExtractListItemRule) Apply(_ AgentInfo, ast *AST) error { node, found := Lookup(ast, r.Path) if !found { return nil @@ -740,7 +808,7 @@ type RenameRule struct { // Apply renames the last items of a Selector to a new name and keep all the other values and will // return an error on failure. -func (r *RenameRule) Apply(ast *AST) error { +func (r *RenameRule) Apply(_ AgentInfo, ast *AST) error { // Skip rename when node is not found. node, ok := Lookup(ast, r.From) if !ok { @@ -773,7 +841,7 @@ func Copy(from, to Selector) *CopyRule { } // Apply copy a part of a tree into a new destination. -func (r CopyRule) Apply(ast *AST) error { +func (r CopyRule) Apply(_ AgentInfo, ast *AST) error { node, ok := Lookup(ast, r.From) // skip when the `from` node is not found. if !ok { @@ -800,7 +868,7 @@ func Translate(path Selector, mapper map[string]interface{}) *TranslateRule { } // Apply translates matching elements of a translation table for a specific selector. -func (r *TranslateRule) Apply(ast *AST) error { +func (r *TranslateRule) Apply(_ AgentInfo, ast *AST) error { // Skip translate when node is not found. node, ok := Lookup(ast, r.Path) if !ok { @@ -873,7 +941,7 @@ func TranslateWithRegexp(path Selector, re *regexp.Regexp, with string) *Transla } // Apply translates matching elements of a translation table for a specific selector. -func (r *TranslateWithRegexpRule) Apply(ast *AST) error { +func (r *TranslateWithRegexpRule) Apply(_ AgentInfo, ast *AST) error { // Skip translate when node is not found. node, ok := Lookup(ast, r.Path) if !ok { @@ -914,7 +982,7 @@ func Map(path Selector, rules ...Rule) *MapRule { } // Apply maps multiples rules over a subset of the tree. -func (r *MapRule) Apply(ast *AST) error { +func (r *MapRule) Apply(agentInfo AgentInfo, ast *AST) error { node, ok := Lookup(ast, r.Path) // Skip map when node is not found. if !ok { @@ -931,15 +999,15 @@ func (r *MapRule) Apply(ast *AST) error { switch t := n.Value().(type) { case *List: - return mapList(r, t) + return mapList(agentInfo, r, t) case *Dict: - return mapDict(r, t) + return mapDict(agentInfo, r, t) case *Key: switch t := n.Value().(type) { case *List: - return mapList(r, t) + return mapList(agentInfo, r, t) case *Dict: - return mapDict(r, t) + return mapDict(agentInfo, r, t) default: return fmt.Errorf( "cannot iterate over node, invalid type expected 'List' or 'Dict' received '%T'", @@ -954,13 +1022,13 @@ func (r *MapRule) Apply(ast *AST) error { ) } -func mapList(r *MapRule, l *List) error { +func mapList(agentInfo AgentInfo, r *MapRule, l *List) error { values := l.Value().([]Node) for idx, item := range values { newAST := &AST{root: item} for _, rule := range r.Rules { - err := rule.Apply(newAST) + err := rule.Apply(agentInfo, newAST) if err != nil { return err } @@ -970,10 +1038,10 @@ func mapList(r *MapRule, l *List) error { return nil } -func mapDict(r *MapRule, l *Dict) error { +func mapDict(agentInfo AgentInfo, r *MapRule, l *Dict) error { newAST := &AST{root: l} for _, rule := range r.Rules { - err := rule.Apply(newAST) + err := rule.Apply(agentInfo, newAST) if err != nil { return err } @@ -1024,7 +1092,7 @@ func Filter(selectors ...Selector) *FilterRule { } // Apply filters a Tree based on list of selectors. -func (r *FilterRule) Apply(ast *AST) error { +func (r *FilterRule) Apply(_ AgentInfo, ast *AST) error { mergedAST := &AST{root: &Dict{}} var err error for _, selector := range r.Selectors { @@ -1054,7 +1122,7 @@ func FilterValues(selector Selector, key Selector, values ...interface{}) *Filte } // Apply filters a Tree based on list of selectors. -func (r *FilterValuesRule) Apply(ast *AST) error { +func (r *FilterValuesRule) Apply(_ AgentInfo, ast *AST) error { node, ok := Lookup(ast, r.Selector) // Skip map when node is not found. if !ok { @@ -1167,7 +1235,7 @@ func (r *FilterValuesWithRegexpRule) UnmarshalYAML(unmarshal func(interface{}) e } // Apply filters a Tree based on list of selectors. -func (r *FilterValuesWithRegexpRule) Apply(ast *AST) error { +func (r *FilterValuesWithRegexpRule) Apply(_ AgentInfo, ast *AST) error { node, ok := Lookup(ast, r.Selector) // Skip map when node is not found. if !ok { diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go index c3207f48cea..d92ba0de985 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/rules_test.go @@ -165,6 +165,51 @@ inputs: }, }, + "inject agent info": { + givenYAML: ` +inputs: + - name: No processors + type: file + - name: With processors + type: file + processors: + - add_fields: + target: other + fields: + data: more +`, + expectedYAML: ` +inputs: + - name: No processors + type: file + processors: + - add_fields: + target: elastic + fields: + agent.id: agent-id + agent.snapshot: false + agent.version: 8.0.0 + - name: With processors + type: file + processors: + - add_fields: + target: other + fields: + data: more + - add_fields: + target: elastic + fields: + agent.id: agent-id + agent.snapshot: false + agent.version: 8.0.0 +`, + rule: &RuleList{ + Rules: []Rule{ + InjectAgentInfo(), + }, + }, + }, + "extract items from array": { givenYAML: ` streams: @@ -615,7 +660,7 @@ logs: a, err := makeASTFromYAML(test.givenYAML) require.NoError(t, err) - err = test.rule.Apply(a) + err = test.rule.Apply(FakeAgentInfo(), a) require.NoError(t, err) v := &MapVisitor{} @@ -751,3 +796,21 @@ func TestSerialization(t *testing.T) { assert.Equal(t, value, v) }) } + +type fakeAgentInfo struct{} + +func (*fakeAgentInfo) AgentID() string { + return "agent-id" +} + +func (*fakeAgentInfo) Version() string { + return "8.0.0" +} + +func (*fakeAgentInfo) Snapshot() bool { + return false +} + +func FakeAgentInfo() AgentInfo { + return &fakeAgentInfo{} +} diff --git a/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go index 33397a87e1e..663484130f4 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/composed/verifier.go @@ -29,11 +29,12 @@ func NewVerifier(verifiers ...download.Verifier) *Verifier { } // Verify checks the package from configured source. -func (e *Verifier) Verify(programName, version string) (bool, error) { +func (e *Verifier) Verify(programName, version, artifactName string, removeOnFailure bool) (bool, error) { var err error - for _, v := range e.vv { - b, e := v.Verify(programName, version) + for i, v := range e.vv { + isLast := (i + 1) == len(e.vv) + b, e := v.Verify(programName, version, artifactName, isLast && removeOnFailure) if e == nil { return b, nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go index d934b20faef..56652d4f69c 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier.go @@ -51,20 +51,23 @@ func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Veri // Verify checks downloaded package on preconfigured // location agains a key stored on elastic.co website. -func (v *Verifier) Verify(programName, version string) (bool, error) { +func (v *Verifier) Verify(programName, version, artifactName string, removeOnFailure bool) (isMatch bool, err error) { filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) if err != nil { return false, errors.New(err, "retrieving package name") } fullPath := filepath.Join(v.config.TargetDirectory, filename) + defer func() { + if removeOnFailure && (!isMatch || err != nil) { + // remove bits so they can be redownloaded + os.Remove(fullPath) + os.Remove(fullPath + ".sha512") + os.Remove(fullPath + ".asc") + } + }() - isMatch, err := v.verifyHash(filename, fullPath) - if !isMatch || err != nil { - // remove bits so they can be redownloaded - os.Remove(fullPath) - os.Remove(fullPath + ".sha512") - os.Remove(fullPath + ".asc") + if isMatch, err := v.verifyHash(filename, fullPath); !isMatch || err != nil { return isMatch, err } diff --git a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go index 4fd845482c5..a79f35bdd6f 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/fs/verifier_test.go @@ -65,7 +65,7 @@ func TestFetchVerify(t *testing.T) { // first download verify should fail: // download skipped, as invalid package is prepared upfront // verify fails and cleans download - matches, err := verifier.Verify(programName, version) + matches, err := verifier.Verify(programName, version, artifactName, true) assert.NoError(t, err) assert.Equal(t, false, matches) @@ -88,7 +88,7 @@ func TestFetchVerify(t *testing.T) { _, err = os.Stat(hashTargetFilePath) assert.NoError(t, err) - matches, err = verifier.Verify(programName, version) + matches, err = verifier.Verify(programName, version, artifactName, true) assert.NoError(t, err) assert.Equal(t, true, matches) } @@ -162,7 +162,7 @@ func TestVerify(t *testing.T) { t.Fatal(err) } - isOk, err := testVerifier.Verify(beatName, version) + isOk, err := testVerifier.Verify(beatName, version, artifactName, true) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go index 0edb979a320..0ff4dfd2b93 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/elastic_test.go @@ -110,7 +110,7 @@ func TestVerify(t *testing.T) { t.Fatal(err) } - isOk, err := testVerifier.Verify(beatName, version) + isOk, err := testVerifier.Verify(beatName, version, artifactName, false) if err != nil { t.Fatal(err) } diff --git a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go index 9f2eacd9395..b5b5e628b4a 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/http/verifier.go @@ -59,9 +59,7 @@ func NewVerifier(config *artifact.Config, allowEmptyPgp bool, pgp []byte) (*Veri // Verify checks downloaded package on preconfigured // location agains a key stored on elastic.co website. -func (v *Verifier) Verify(programName, version string) (bool, error) { - // TODO: think about verifying asc for prepacked beats - +func (v *Verifier) Verify(programName, version, artifactName string, removeOnFailure bool) (isMatch bool, err error) { filename, err := artifact.GetArtifactName(programName, version, v.config.OS(), v.config.Arch()) if err != nil { return false, errors.New(err, "retrieving package name") @@ -72,16 +70,20 @@ func (v *Verifier) Verify(programName, version string) (bool, error) { return false, errors.New(err, "retrieving package path") } - isMatch, err := v.verifyHash(filename, fullPath) - if !isMatch || err != nil { - // remove bits so they can be redownloaded - os.Remove(fullPath) - os.Remove(fullPath + ".sha512") - os.Remove(fullPath + ".asc") + defer func() { + if removeOnFailure && (!isMatch || err != nil) { + // remove bits so they can be redownloaded + os.Remove(fullPath) + os.Remove(fullPath + ".sha512") + os.Remove(fullPath + ".asc") + } + }() + + if isMatch, err := v.verifyHash(filename, fullPath); !isMatch || err != nil { return isMatch, err } - return v.verifyAsc(programName, version) + return v.verifyAsc(programName, version, artifactName) } func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { @@ -127,7 +129,7 @@ func (v *Verifier) verifyHash(filename, fullPath string) (bool, error) { return expectedHash == computedHash, nil } -func (v *Verifier) verifyAsc(programName, version string) (bool, error) { +func (v *Verifier) verifyAsc(programName, version, artifactName string) (bool, error) { if len(v.pgpBytes) == 0 { // no pgp available skip verification process return true, nil @@ -143,7 +145,7 @@ func (v *Verifier) verifyAsc(programName, version string) (bool, error) { return false, errors.New(err, "retrieving package path") } - ascURI, err := v.composeURI(programName, filename) + ascURI, err := v.composeURI(filename, artifactName) if err != nil { return false, errors.New(err, "composing URI for fetching asc file", errors.TypeNetwork) } @@ -177,7 +179,7 @@ func (v *Verifier) verifyAsc(programName, version string) (bool, error) { } -func (v *Verifier) composeURI(programName, filename string) (string, error) { +func (v *Verifier) composeURI(filename, artifactName string) (string, error) { upstream := v.config.SourceURI if !strings.HasPrefix(upstream, "http") && !strings.HasPrefix(upstream, "file") && !strings.HasPrefix(upstream, "/") { // always default to https @@ -190,7 +192,7 @@ func (v *Verifier) composeURI(programName, filename string) (string, error) { return "", errors.New(err, "invalid upstream URI", errors.TypeNetwork, errors.M(errors.MetaKeyURI, upstream)) } - uri.Path = path.Join(uri.Path, "beats", programName, filename+ascSuffix) + uri.Path = path.Join(uri.Path, artifactName, filename+ascSuffix) return uri.String(), nil } diff --git a/x-pack/elastic-agent/pkg/artifact/download/verifier.go b/x-pack/elastic-agent/pkg/artifact/download/verifier.go index 6aa4dc4abe4..d7b2bc7a415 100644 --- a/x-pack/elastic-agent/pkg/artifact/download/verifier.go +++ b/x-pack/elastic-agent/pkg/artifact/download/verifier.go @@ -6,5 +6,5 @@ package download // Verifier is an interface verifying GPG key of a downloaded artifact type Verifier interface { - Verify(programName, version string) (bool, error) + Verify(programName, version, artifactName string, removeOnFailure bool) (bool, error) } diff --git a/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go new file mode 100644 index 00000000000..5e26436bfc4 --- /dev/null +++ b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer.go @@ -0,0 +1,62 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package atomic + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" +) + +type embeddedInstaller interface { + Install(ctx context.Context, programName, version, installDir string) error +} + +// Installer installs into temporary destination and moves to correct one after +// successful finish. +type Installer struct { + installer embeddedInstaller +} + +// NewInstaller creates a new AtomicInstaller +func NewInstaller(i embeddedInstaller) (*Installer, error) { + return &Installer{ + installer: i, + }, nil +} + +// Install performs installation of program in a specific version. +func (i *Installer) Install(ctx context.Context, programName, version, installDir string) error { + // tar installer uses Dir of installDir to determine location of unpack + tempDir, err := ioutil.TempDir(os.TempDir(), "elastic-agent-install") + if err != nil { + return err + } + tempInstallDir := filepath.Join(tempDir, filepath.Base(installDir)) + + // cleanup install directory before Install + if _, err := os.Stat(installDir); err == nil || os.IsExist(err) { + os.RemoveAll(installDir) + } + + if _, err := os.Stat(tempInstallDir); err == nil || os.IsExist(err) { + os.RemoveAll(tempInstallDir) + } + + if err := i.installer.Install(ctx, programName, version, tempInstallDir); err != nil { + // cleanup unfinished install + os.RemoveAll(tempInstallDir) + return err + } + + if err := os.Rename(tempInstallDir, installDir); err != nil { + os.RemoveAll(installDir) + os.RemoveAll(tempInstallDir) + return err + } + + return nil +} diff --git a/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer_test.go b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer_test.go new file mode 100644 index 00000000000..d6266659b7d --- /dev/null +++ b/x-pack/elastic-agent/pkg/artifact/install/atomic/atomic_installer_test.go @@ -0,0 +1,115 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package atomic + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOKInstall(t *testing.T) { + sig := make(chan int) + ti := &testInstaller{sig} + var wg sync.WaitGroup + i, err := NewInstaller(ti) + + assert.NoError(t, err) + + ctx := context.Background() + installDir := filepath.Join(os.TempDir(), "install_dir") + + wg.Add(1) + go func() { + err := i.Install(ctx, "a", "b", installDir) + assert.NoError(t, err) + wg.Done() + }() + + // signal to process next files + close(sig) + + wg.Wait() + + assert.DirExists(t, installDir) + files := getFiles() + + for name := range files { + path := filepath.Join(installDir, name) + assert.FileExists(t, path) + } + + os.RemoveAll(installDir) +} + +func TestContextCancelledInstall(t *testing.T) { + sig := make(chan int) + ti := &testInstaller{sig} + var wg sync.WaitGroup + i, err := NewInstaller(ti) + + assert.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + installDir := filepath.Join(os.TempDir(), "install_dir") + + wg.Add(1) + go func() { + err := i.Install(ctx, "a", "b", installDir) + assert.Error(t, err) + wg.Done() + }() + + // cancel before signaling + cancel() + close(sig) + + wg.Wait() + + assert.NoDirExists(t, installDir) +} + +type testInstaller struct { + signal chan int +} + +func (ti *testInstaller) Install(ctx context.Context, programName, version, installDir string) error { + files := getFiles() + if err := os.MkdirAll(installDir, 0777); err != nil { + return err + } + + for name, content := range files { + if err := ctx.Err(); err != nil { + return err + } + + filename := filepath.Join(installDir, name) + if err := ioutil.WriteFile(filename, content, 0666); err != nil { + return err + } + + // wait for all but last + <-ti.signal + } + + return nil +} + +func getFiles() map[string][]byte { + files := make(map[string][]byte) + fileCount := 3 + for i := 1; i <= fileCount; i++ { + files[fmt.Sprintf("file_%d", i)] = []byte(fmt.Sprintf("content of file %d", i)) + } + + return files +} diff --git a/x-pack/elastic-agent/pkg/artifact/install/installer.go b/x-pack/elastic-agent/pkg/artifact/install/installer.go index f04e7a4238e..c606ada5d65 100644 --- a/x-pack/elastic-agent/pkg/artifact/install/installer.go +++ b/x-pack/elastic-agent/pkg/artifact/install/installer.go @@ -12,6 +12,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/install/dir" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/install/atomic" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/install/hooks" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/install/tar" "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/artifact/install/zip" @@ -60,5 +61,10 @@ func NewInstaller(config *artifact.Config) (InstallerChecker, error) { return nil, err } - return hooks.NewInstallerChecker(installer, dir.NewChecker()) + atomicInstaller, err := atomic.NewInstaller(installer) + if err != nil { + return nil, err + } + + return hooks.NewInstallerChecker(atomicInstaller, dir.NewChecker()) } diff --git a/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go b/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go index 5c7f0f593a3..74a74e4c6bc 100644 --- a/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go +++ b/x-pack/elastic-agent/pkg/artifact/install/tar/tar_installer.go @@ -99,9 +99,19 @@ func unpack(r io.Reader, dir string) error { } _, err = io.Copy(wf, tr) + + if err == nil { + // sometimes we try executing binary too fast and run into text file busy after unpacking + // syncing prevents this + if syncErr := wf.Sync(); syncErr != nil { + err = syncErr + } + } + if closeErr := wf.Close(); closeErr != nil && err == nil { err = closeErr } + if err != nil { return fmt.Errorf("TarInstaller: error writing to %s: %v", abs, err) } diff --git a/x-pack/elastic-agent/pkg/artifact/install/zip/zip_installer.go b/x-pack/elastic-agent/pkg/artifact/install/zip/zip_installer.go index ffc90f2dce8..b565f630a73 100644 --- a/x-pack/elastic-agent/pkg/artifact/install/zip/zip_installer.go +++ b/x-pack/elastic-agent/pkg/artifact/install/zip/zip_installer.go @@ -102,14 +102,18 @@ func (i *Installer) unzip(artifactPath string) error { return err } defer func() { - if cerr := f.Close(); cerr != nil { - err = multierror.Append(err, cerr) + if closeErr := f.Close(); closeErr != nil { + err = multierror.Append(err, closeErr) } }() if _, err = io.Copy(f, rc); err != nil { return err } + + // sometimes we try executing binary too fast and run into text file busy after unpacking + // syncing prevents this + f.Sync() } return nil } diff --git a/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go b/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go index 119809338d6..6c656839820 100644 --- a/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go +++ b/x-pack/elastic-agent/pkg/basecmd/version/cmd_test.go @@ -52,7 +52,7 @@ func TestCmdBinaryOnlyYAML(t *testing.T) { } func TestCmdDaemon(t *testing.T) { - srv := server.New(newErrorLogger(t), nil) + srv := server.New(newErrorLogger(t), nil, nil) require.NoError(t, srv.Start()) defer srv.Stop() @@ -67,7 +67,7 @@ func TestCmdDaemon(t *testing.T) { } func TestCmdDaemonYAML(t *testing.T) { - srv := server.New(newErrorLogger(t), nil) + srv := server.New(newErrorLogger(t), nil, nil) require.NoError(t, srv.Start()) defer srv.Stop() diff --git a/x-pack/elastic-agent/pkg/composable/providers/kubernetes/config.go b/x-pack/elastic-agent/pkg/composable/providers/kubernetes/config.go new file mode 100644 index 00000000000..200bedbe878 --- /dev/null +++ b/x-pack/elastic-agent/pkg/composable/providers/kubernetes/config.go @@ -0,0 +1,31 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// TODO review the need for this +// +build linux darwin windows + +package kubernetes + +import ( + "time" +) + +// Config for kubernetes provider +type Config struct { + KubeConfig string `config:"kube_config"` + SyncPeriod time.Duration `config:"sync_period"` + CleanupTimeout time.Duration `config:"cleanup_timeout" validate:"positive"` + + // Needed when resource is a pod + Node string `config:"node"` + + // Scope of the provider (cluster or node) + Scope string `config:"scope"` +} + +// InitDefaults initializes the default values for the config. +func (c *Config) InitDefaults() { + c.SyncPeriod = 10 * time.Minute + c.CleanupTimeout = 60 * time.Second +} diff --git a/x-pack/elastic-agent/pkg/composable/providers/kubernetes/kubernetes.go b/x-pack/elastic-agent/pkg/composable/providers/kubernetes/kubernetes.go new file mode 100644 index 00000000000..c777c8a05ce --- /dev/null +++ b/x-pack/elastic-agent/pkg/composable/providers/kubernetes/kubernetes.go @@ -0,0 +1,215 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package kubernetes + +import ( + "fmt" + "time" + + "github.com/elastic/beats/v7/libbeat/common/kubernetes" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/composable" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config" + "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" +) + +func init() { + composable.Providers.AddDynamicProvider("kubernetes", DynamicProviderBuilder) +} + +type dynamicProvider struct { + logger *logger.Logger + config *Config +} + +type eventWatcher struct { + logger *logger.Logger + cleanupTimeout time.Duration + comm composable.DynamicProviderComm +} + +// DynamicProviderBuilder builds the dynamic provider. +func DynamicProviderBuilder(logger *logger.Logger, c *config.Config) (composable.DynamicProvider, error) { + var cfg Config + if c == nil { + c = config.New() + } + err := c.Unpack(&cfg) + if err != nil { + return nil, errors.New(err, "failed to unpack configuration") + } + return &dynamicProvider{logger, &cfg}, nil +} + +// Run runs the environment context provider. +func (p *dynamicProvider) Run(comm composable.DynamicProviderComm) error { + client, err := kubernetes.GetKubernetesClient(p.config.KubeConfig) + if err != nil { + // info only; return nil (do nothing) + p.logger.Debugf("Kubernetes provider skipped, unable to connect: %s", err) + return nil + } + + // Ensure that node is set correctly whenever the scope is set to "node". Make sure that node is empty + // when cluster scope is enforced. + if p.config.Scope == "node" { + p.config.Node = kubernetes.DiscoverKubernetesNode(p.logger, p.config.Node, kubernetes.IsInCluster(p.config.KubeConfig), client) + } else { + p.config.Node = "" + } + p.logger.Infof("Kubernetes provider started with %s scope", p.config.Scope) + p.logger.Debugf("Initializing Kubernetes watcher using node: %v", p.config.Node) + + watcher, err := kubernetes.NewWatcher(client, &kubernetes.Pod{}, kubernetes.WatchOptions{ + SyncTimeout: p.config.SyncPeriod, + Node: p.config.Node, + //Namespace: p.config.Namespace, + }, nil) + if err != nil { + return errors.New(err, "couldn't create kubernetes watcher") + } + + watcher.AddEventHandler(&eventWatcher{p.logger, p.config.CleanupTimeout, comm}) + + err = watcher.Start() + if err != nil { + return errors.New(err, "couldn't start kubernetes watcher") + } + + return nil +} + +func (p *eventWatcher) emitRunning(pod *kubernetes.Pod) { + mapping := map[string]interface{}{ + "namespace": pod.GetNamespace(), + "pod": map[string]interface{}{ + "uid": string(pod.GetUID()), + "name": pod.GetName(), + "labels": pod.GetLabels(), + "ip": pod.Status.PodIP, + }, + } + + processors := []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": mapping, + "target": "kubernetes", + }, + }, + } + + // Emit the pod + // We emit Pod + containers to ensure that configs matching Pod only + // get Pod metadata (not specific to any container) + p.comm.AddOrUpdate(string(pod.GetUID()), mapping, processors) + + // Emit all containers in the pod + p.emitContainers(pod, pod.Spec.Containers, pod.Status.ContainerStatuses) + + // TODO deal with init containers stopping after initialization + p.emitContainers(pod, pod.Spec.InitContainers, pod.Status.InitContainerStatuses) +} + +func (p *eventWatcher) emitContainers(pod *kubernetes.Pod, containers []kubernetes.Container, containerstatuses []kubernetes.PodContainerStatus) { + // Collect all runtimes from status information. + containerIDs := map[string]string{} + runtimes := map[string]string{} + for _, c := range containerstatuses { + cid, runtime := kubernetes.ContainerIDWithRuntime(c) + containerIDs[c.Name] = cid + runtimes[c.Name] = runtime + } + + for _, c := range containers { + // If it doesn't have an ID, container doesn't exist in + // the runtime, emit only an event if we are stopping, so + // we are sure of cleaning up configurations. + cid := containerIDs[c.Name] + if cid == "" { + continue + } + + // ID is the combination of pod UID + container name + eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name) + + mapping := map[string]interface{}{ + "namespace": pod.GetNamespace(), + "pod": map[string]interface{}{ + "uid": string(pod.GetUID()), + "name": pod.GetName(), + "labels": pod.GetLabels(), + "ip": pod.Status.PodIP, + }, + "container": map[string]interface{}{ + "id": cid, + "name": c.Name, + "image": c.Image, + "runtime": runtimes[c.Name], + }, + } + + processors := []map[string]interface{}{ + { + "add_fields": map[string]interface{}{ + "fields": mapping, + "target": "kubernetes", + }, + }, + } + + // Emit the container + p.comm.AddOrUpdate(eventID, mapping, processors) + } +} + +func (p *eventWatcher) emitStopped(pod *kubernetes.Pod) { + p.comm.Remove(string(pod.GetUID())) + + for _, c := range pod.Spec.Containers { + // ID is the combination of pod UID + container name + eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name) + p.comm.Remove(eventID) + } + + for _, c := range pod.Spec.InitContainers { + // ID is the combination of pod UID + container name + eventID := fmt.Sprintf("%s.%s", pod.GetObjectMeta().GetUID(), c.Name) + p.comm.Remove(eventID) + } +} + +// OnAdd ensures processing of pod objects that are newly added +func (p *eventWatcher) OnAdd(obj interface{}) { + p.logger.Debugf("pod add: %+v", obj) + p.emitRunning(obj.(*kubernetes.Pod)) +} + +// OnUpdate emits events for a given pod depending on the state of the pod, +// if it is terminating, a stop event is scheduled, if not, a stop and a start +// events are sent sequentially to recreate the resources assotiated to the pod. +func (p *eventWatcher) OnUpdate(obj interface{}) { + pod := obj.(*kubernetes.Pod) + + p.logger.Debugf("pod update for pod: %+v, status: %+v", pod.Name, pod.Status.Phase) + switch pod.Status.Phase { + case kubernetes.PodSucceeded, kubernetes.PodFailed: + time.AfterFunc(p.cleanupTimeout, func() { p.emitStopped(pod) }) + return + case kubernetes.PodPending: + p.logger.Debugf("pod update (pending): don't know what to do with this pod yet, skipping for now: %+v", obj) + return + } + + p.logger.Debugf("pod update: %+v", obj) + p.emitRunning(pod) +} + +// OnDelete stops pod objects that are deleted +func (p *eventWatcher) OnDelete(obj interface{}) { + p.logger.Debugf("pod delete: %+v", obj) + pod := obj.(*kubernetes.Pod) + time.AfterFunc(p.cleanupTimeout, func() { p.emitStopped(pod) }) +} diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/app.go b/x-pack/elastic-agent/pkg/core/plugin/process/app.go index 5f9cf6efb25..8732af24f68 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/app.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/app.go @@ -117,7 +117,7 @@ func (a *Application) Name() string { // Started returns true if the application is started. func (a *Application) Started() bool { - return a.state.Status != state.Stopped + return a.state.Status != state.Stopped && a.state.Status != state.Crashed && a.state.Status != state.Failed } // Stop stops the current application. diff --git a/x-pack/elastic-agent/pkg/core/plugin/process/start.go b/x-pack/elastic-agent/pkg/core/plugin/process/start.go index c5f1ffb3842..bc19910e53c 100644 --- a/x-pack/elastic-agent/pkg/core/plugin/process/start.go +++ b/x-pack/elastic-agent/pkg/core/plugin/process/start.go @@ -39,7 +39,7 @@ func (a *Application) start(ctx context.Context, t app.Taggable, cfg map[string] }() // already started if not stopped or crashed - if a.state.Status != state.Stopped && a.state.Status != state.Crashed && a.state.Status != state.Failed { + if a.Started() { return nil } diff --git a/x-pack/elastic-agent/pkg/core/state/state.go b/x-pack/elastic-agent/pkg/core/state/state.go index 6b7c8bd53de..670cdc2a2f2 100644 --- a/x-pack/elastic-agent/pkg/core/state/state.go +++ b/x-pack/elastic-agent/pkg/core/state/state.go @@ -30,6 +30,8 @@ const ( Crashed // Restarting is status describing application is restarting. Restarting + // Updating is status describing application is updating. + Updating ) // State wraps the process state and application status. diff --git a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go index ac568c884d1..5c197289ed9 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd.go @@ -14,7 +14,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) -const ackPath = "/api/ingest_manager/fleet/agents/%s/acks" +const ackPath = "/api/fleet/agents/%s/acks" // AckEvent is an event sent in an ACK request. type AckEvent struct { diff --git a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go index 75940e928b7..1f1bfdb21eb 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/ack_cmd_test.go @@ -22,7 +22,7 @@ func TestAck(t *testing.T) { func(t *testing.T) *http.ServeMux { raw := `{"action": "ack"}` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/acks", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/acks", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go index 58c80b0adf1..9758fd4236c 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd.go @@ -17,7 +17,7 @@ import ( "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" ) -const checkingPath = "/api/ingest_manager/fleet/agents/%s/checkin" +const checkingPath = "/api/fleet/agents/%s/checkin" // CheckinRequest consists of multiple events reported to fleet ui. type CheckinRequest struct { diff --git a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go index af8b4da81ea..3e88ed29cd1 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/checkin_cmd_test.go @@ -34,7 +34,7 @@ func TestCheckin(t *testing.T) { } ` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) fmt.Fprintf(w, raw) @@ -83,7 +83,7 @@ func TestCheckin(t *testing.T) { } ` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, raw) @@ -144,7 +144,7 @@ func TestCheckin(t *testing.T) { } ` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, raw) @@ -176,7 +176,7 @@ func TestCheckin(t *testing.T) { func(t *testing.T) *http.ServeMux { raw := `{ "actions": [] }` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) fmt.Fprintf(w, raw) @@ -199,7 +199,7 @@ func TestCheckin(t *testing.T) { func(t *testing.T) *http.ServeMux { raw := `{"actions": []}` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { type Request struct { Metadata *info.ECSMeta `json:"local_metadata"` @@ -233,7 +233,7 @@ func TestCheckin(t *testing.T) { func(t *testing.T) *http.ServeMux { raw := `{"actions": []}` mux := http.NewServeMux() - path := fmt.Sprintf("/api/ingest_manager/fleet/agents/%s/checkin", agentInfo.AgentID()) + path := fmt.Sprintf("/api/fleet/agents/%s/checkin", agentInfo.AgentID()) mux.HandleFunc(path, authHandler(func(w http.ResponseWriter, r *http.Request) { type Request struct { Metadata *info.ECSMeta `json:"local_metadata"` diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go index 3e6336fbc81..29168be0ae4 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd.go @@ -71,7 +71,7 @@ func (p EnrollType) MarshalJSON() ([]byte, error) { // EnrollRequest is the data required to enroll the elastic-agent into Fleet. // // Example: -// POST /api/ingest_manager/fleet/agents/enroll +// POST /api/fleet/agents/enroll // { // "type": "PERMANENT", // "metadata": { @@ -168,7 +168,7 @@ type EnrollCmd struct { // Execute enroll the Agent in the Fleet. func (e *EnrollCmd) Execute(ctx context.Context, r *EnrollRequest) (*EnrollResponse, error) { - const p = "/api/ingest_manager/fleet/agents/enroll" + const p = "/api/fleet/agents/enroll" const key = "Authorization" const prefix = "ApiKey " diff --git a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go index 6533ad6643e..29bbf7e607e 100644 --- a/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go +++ b/x-pack/elastic-agent/pkg/fleetapi/enroll_cmd_test.go @@ -23,7 +23,7 @@ func TestEnroll(t *testing.T) { t.Run("Successful enroll", withServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") @@ -92,7 +92,7 @@ func TestEnroll(t *testing.T) { t.Run("Raise back any server errors", withServer( func(t *testing.T) *http.ServeMux { mux := http.NewServeMux() - mux.HandleFunc("/api/ingest_manager/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/api/fleet/agents/enroll", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusInternalServerError) w.Header().Set("Content-Type", "application/json") w.Write([]byte(`{"statusCode": 500, "error":"Something is really bad here"}`)) diff --git a/x-pack/elastic-agent/pkg/reporter/reporter.go b/x-pack/elastic-agent/pkg/reporter/reporter.go index c36708a837f..3b128841b2a 100644 --- a/x-pack/elastic-agent/pkg/reporter/reporter.go +++ b/x-pack/elastic-agent/pkg/reporter/reporter.go @@ -38,6 +38,8 @@ const ( EventSubTypeFailed = "FAILED" // EventSubTypeStopping is an event type indicating application is stopping. EventSubTypeStopping = "STOPPING" + // EventSubTypeUpdating is an event type indicating update process in progress. + EventSubTypeUpdating = "UPDATING" ) type agentInfo interface { @@ -127,6 +129,10 @@ func generateRecord(agentID string, id string, name string, s state.State) event case state.Restarting: subType = EventSubTypeStarting subTypeText = "RESTARTING" + case state.Updating: + subType = EventSubTypeUpdating + subTypeText = EventSubTypeUpdating + } err := errors.New( diff --git a/x-pack/elastic-agent/spec/filebeat.yml b/x-pack/elastic-agent/spec/filebeat.yml index 1b184b10098..aa09b4f9121 100644 --- a/x-pack/elastic-agent/spec/filebeat.yml +++ b/x-pack/elastic-agent/spec/filebeat.yml @@ -87,6 +87,8 @@ rules: values: - true +- inject_agent_info: {} + - copy: from: inputs to: filebeat diff --git a/x-pack/elastic-agent/spec/metricbeat.yml b/x-pack/elastic-agent/spec/metricbeat.yml index 94b69e9a2f3..a5015a974a5 100644 --- a/x-pack/elastic-agent/spec/metricbeat.yml +++ b/x-pack/elastic-agent/spec/metricbeat.yml @@ -73,6 +73,8 @@ rules: - remove_key: key: use_output +- inject_agent_info: {} + - copy: from: inputs to: metricbeat diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 8282f9a02b0..d8cef3dd206 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -46,3 +46,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/filebeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc index 8891e38fcc4..5cbe4685cb8 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc @@ -42,6 +42,11 @@ The `s3` input supports the following configuration options plus the URL of the AWS SQS queue that messages will be received from. Required. +[float] +==== `fips_enabled` + +Enabling this option changes the service name from `s3` to `s3-fips` for connecting to the correct service endpoint. For example: `s3-fips.us-gov-east-1.amazonaws.com`. + [float] ==== `visibility_timeout` diff --git a/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc b/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc index f8d5dd51015..551d0095b3a 100644 --- a/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc @@ -23,6 +23,7 @@ Example configurations: api_address: https://api.dev.cfdev.sh client_id: uaa-filebeat client_secret: verysecret + shard_id: filebeat ssl: verification_mode: none ---- @@ -34,6 +35,7 @@ Example configurations: api_address: https://api.dev.cfdev.sh client_id: uaa-filebeat client_secret: verysecret + shard_id: filebeat ssl.certificate_authorities: ["/etc/pki/cf/ca.pem"] ssl.certificate: "/etc/pki/cf/cert.pem" ssl.key: "/etc/pki/cf/cert.key" diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 9797291bdf4..49ede1c7d24 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1050,6 +1050,19 @@ filebeat.modules: # "+02:00" for GMT+02:00 # var.tz_offset: local + srx: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9006. + #var.syslog_port: 9006 + #-------------------------------- Kafka Module -------------------------------- - module: kafka # All logs @@ -1105,7 +1118,20 @@ filebeat.modules: # Oauth Client Secret #var.oauth2.client.secret: "" - + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + m365_defender: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + # Oauth Token URL, should include the tenant ID #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" dhcp: @@ -1757,6 +1783,7 @@ filebeat.inputs: # # Possible options are: # * log: Reads every line of the log file (default) +# * filestream: Improved version of log input. Experimental. # * stdin: Reads the standard in #------------------------------ Log input -------------------------------- @@ -1977,6 +2004,145 @@ filebeat.inputs: # Defines if inputs is enabled #enabled: true +#--------------------------- Filestream input ---------------------------- +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + # To fetch all ".log" files from a specific level of subdirectories + # /var/log/*/*.log can be used. + # For each file found under this path, a harvester is started. + # Make sure not file is defined twice as this can lead to unexpected behaviour. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Configure the file encoding for reading files with international characters + # following the W3C recommendation for HTML5 (http://www.w3.org/TR/encoding). + # Some sample encodings: + # plain, utf-8, utf-16be-bom, utf-16be, utf-16le, big5, gb18030, gbk, + # hz-gb-2312, euc-kr, euc-jp, iso-2022-jp, shift-jis, ... + #encoding: plain + + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, no lines are dropped. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. The include_lines is called before + # exclude_lines. By default, all the lines are exported. + #include_lines: ['^ERR', '^WARN'] + + ### Prospector options + + # How often the input checks for new files in the paths that are specified + # for harvesting. Specify 1s to scan the directory as frequently as possible + # without causing Filebeat to scan too frequently. Default: 10s. + #prospector.scanner.check_interval: 10s + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Expand "**" patterns into regular glob patterns. + #prospector.scanner.recursive_glob: true + + # If symlinks is enabled, symlinks are opened and harvested. The harvester is opening the + # original for harvesting but will report the symlink name as source. + #prospector.scanner.symlinks: false + + ### State options + + # Files for the modification data is older then clean_inactive the state from the registry is removed + # By default this is disabled. + #clean_inactive: 0 + + # Removes the state for file which cannot be found on disk anymore immediately + #clean_removed: true + + # Method to determine if two files are the same or not. By default + # the Beat considers two files the same if their inode and device id are the same. + #file_identity.native: ~ + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + + # Set to true to publish fields with null values in events. + #keep_null: false + + # By default, all events contain `host.name`. This option can be set to true + # to disable the addition of this field to all events. The default value is + # false. + #publisher_pipeline.disable_host: false + + # Ignore files which were modified more then the defined timespan in the past. + # ignore_older is disabled by default, so no files are ignored by setting it to 0. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #ignore_older: 0 + + # Defines the buffer size every harvester uses when fetching the file + #harvester_buffer_size: 16384 + + # Maximum number of bytes a single log event can have + # All bytes after max_bytes are discarded and not sent. The default is 10MB. + # This is especially useful for multiline log messages which can get large. + #message_max_bytes: 10485760 + + # Characters which separate the lines. Valid values: auto, line_feed, vertical_tab, form_feed, + # carriage_return, carriage_return_line_feed, next_line, line_separator, paragraph_separator. + #line_terminator: auto + + # The Ingest Node pipeline ID associated with this input. If this is set, it + # overwrites the pipeline option from the Elasticsearch output. + #pipeline: + + # Backoff values define how aggressively filebeat crawls new files for updates + # The default values can be used in most cases. Backoff defines how long it is waited + # to check a file again after EOF is reached. Default is 1s which means the file + # is checked every second if new lines were added. This leads to a near real time crawling. + # Every time a new line appears, backoff is reset to the initial value. + #backoff.init: 1s + + # Max backoff defines what the maximum backoff time is. After having backed off multiple times + # from checking the files, the waiting time will never exceed max_backoff independent of the + # backoff factor. Having it set to 10s means in the worst case a new line can be added to a log + # file after having backed off multiple times, it takes a maximum of 10s to read the new line + #backoff.max: 10s + + ### Harvester closing options + + # Close inactive closes the file handler after the predefined period. + # The period starts when the last line of the file was, not the file ModTime. + # Time strings like 2h (2 hours), 5m (5 minutes) can be used. + #close.on_state_change.inactive: 5m + + # Close renamed closes a file handler when the file is renamed or rotated. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.on_state_change.renamed: false + + # When enabling this option, a file handler is closed immediately in case a file can't be found + # any more. In case the file shows up again later, harvesting will continue at the last known position + # after scan_frequency. + #close.on_state_change.removed: true + + # Closes the file handler as soon as the harvesters reaches the end of the file. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.eof: false + + # Close timeout closes the harvester after the predefined time. + # This is independent if the harvester did finish reading the file or not. + # By default this option is disabled. + # Note: Potential data loss. Make sure to read and understand the docs for this option. + #close.reader.after_interval: 0 + #----------------------------- Stdin input ------------------------------- # Configuration to use stdin input #- type: stdin diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index 2a3e678c542..9a29bc76962 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -62,6 +62,35 @@ filebeat.inputs: # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash #multiline.match: after +# filestream is an experimental input. It is going to replace log input in the future. +- type: filestream + + # Change to true to enable this input configuration. + enabled: false + + # Paths that should be crawled and fetched. Glob based paths. + paths: + - /var/log/*.log + #- c:\programdata\elasticsearch\logs\* + + # Exclude lines. A list of regular expressions to match. It drops the lines that are + # matching any regular expression from the list. + #exclude_lines: ['^DBG'] + + # Include lines. A list of regular expressions to match. It exports the lines that are + # matching any regular expression from the list. + #include_lines: ['^ERR', '^WARN'] + + # Exclude files. A list of regular expressions to match. Filebeat drops the files that + # are matching any regular expression from the list. By default, no files are dropped. + #prospector.scanner.exclude_files: ['.gz$'] + + # Optional additional fields. These fields can be freely picked + # to add additional information to the crawled log files for filtering + #fields: + # level: debug + # review: 1 + # ============================== Filebeat modules ============================== filebeat.config.modules: diff --git a/x-pack/filebeat/input/cloudfoundry/input.go b/x-pack/filebeat/input/cloudfoundry/input.go index 036a61b9d1e..3d2b9b34e59 100644 --- a/x-pack/filebeat/input/cloudfoundry/input.go +++ b/x-pack/filebeat/input/cloudfoundry/input.go @@ -25,7 +25,7 @@ type cloudfoundryEvent interface { func Plugin() v2.Plugin { return v2.Plugin{ Name: "cloudfoundry", - Stability: feature.Beta, + Stability: feature.Stable, Deprecated: false, Info: "collect logs from cloudfoundry loggregator", Manager: stateless.NewInputManager(configure), diff --git a/x-pack/filebeat/input/default-inputs/inputs.go b/x-pack/filebeat/input/default-inputs/inputs.go index cd8562560da..4779b452f1d 100644 --- a/x-pack/filebeat/input/default-inputs/inputs.go +++ b/x-pack/filebeat/input/default-inputs/inputs.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/beats/v7/x-pack/filebeat/input/http_endpoint" "github.com/elastic/beats/v7/x-pack/filebeat/input/httpjson" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit" + "github.com/elastic/beats/v7/x-pack/filebeat/input/s3" ) func Init(info beat.Info, log *logp.Logger, store beater.StateStore) []v2.Plugin { @@ -29,5 +30,6 @@ func xpackInputs(info beat.Info, log *logp.Logger, store beater.StateStore) []v2 http_endpoint.Plugin(), httpjson.Plugin(log, store), o365audit.Plugin(log, store), + s3.Plugin(), } } diff --git a/x-pack/filebeat/input/httpjson/config_test.go b/x-pack/filebeat/input/httpjson/config_test.go index 85c7c64848d..1986ee7abe9 100644 --- a/x-pack/filebeat/input/httpjson/config_test.go +++ b/x-pack/filebeat/input/httpjson/config_test.go @@ -362,6 +362,7 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + /* Flaky test: https://github.com/elastic/beats/issues/21748 { name: "date_cursor.date_format will fail if invalid", expectedErr: "invalid configuration: date_format is not a valid date layout accessing 'date_cursor'", @@ -370,6 +371,7 @@ func TestConfigOauth2Validation(t *testing.T) { "url": "localhost", }, }, + */ { name: "date_cursor must work with a valid date_format", input: map[string]interface{}{ diff --git a/x-pack/filebeat/input/o365audit/input.go b/x-pack/filebeat/input/o365audit/input.go index 67013dc6a71..2cf76de9ee3 100644 --- a/x-pack/filebeat/input/o365audit/input.go +++ b/x-pack/filebeat/input/o365audit/input.go @@ -21,14 +21,12 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/filebeat/input/o365audit/poll" "github.com/elastic/go-concert/ctxtool" + "github.com/elastic/go-concert/timed" ) const ( pluginName = "o365audit" fieldsPrefix = pluginName - - // How long to retry when a fatal error is encountered in the input. - failureRetryInterval = time.Minute * 5 ) type o365input struct { @@ -126,8 +124,8 @@ func (inp *o365input) Run( } publisher.Publish(event, nil) ctx.Logger.Errorf("Input failed: %v", err) - ctx.Logger.Infof("Restarting in %v", failureRetryInterval) - time.Sleep(failureRetryInterval) + ctx.Logger.Infof("Restarting in %v", inp.config.API.ErrorRetryInterval) + timed.Wait(ctx.Cancelation, inp.config.API.ErrorRetryInterval) } } return nil diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/s3/collector.go new file mode 100644 index 00000000000..1b890513284 --- /dev/null +++ b/x-pack/filebeat/input/s3/collector.go @@ -0,0 +1,633 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package s3 + +import ( + "bufio" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" + + awssdk "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/awserr" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/go-concert/unison" +) + +type s3Collector struct { + cancellation context.Context + logger *logp.Logger + + config *config + visibilityTimeout int64 + + sqs sqsiface.ClientAPI + s3 s3iface.ClientAPI + publisher beat.Client +} + +type s3Info struct { + name string + key string + region string + arn string + expandEventListFromField string +} + +type bucket struct { + Name string `json:"name"` + Arn string `json:"arn"` +} + +type object struct { + Key string `json:"key"` +} + +type s3BucketObject struct { + bucket `json:"bucket"` + object `json:"object"` +} + +type sqsMessage struct { + Records []struct { + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventName string `json:"eventName"` + S3 s3BucketObject `json:"s3"` + } `json:"Records"` +} + +type s3Context struct { + mux sync.Mutex + refs int + err error // first error witnessed or multi error + errC chan error +} + +var ( + // The maximum number of messages to return. Amazon SQS never returns more messages + // than this value (however, fewer messages might be returned). + maxNumberOfMessage uint8 = 10 + + // The duration (in seconds) for which the call waits for a message to arrive + // in the queue before returning. If a message is available, the call returns + // sooner than WaitTimeSeconds. If no messages are available and the wait time + // expires, the call returns successfully with an empty list of messages. + waitTimeSecond uint8 = 10 +) + +func (c *s3Collector) run() { + defer c.logger.Info("s3 input worker has stopped.") + c.logger.Info("s3 input worker has started.") + for c.cancellation.Err() == nil { + // receive messages from sqs + output, err := c.receiveMessage(c.sqs, c.visibilityTimeout) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + continue + } + c.logger.Error("SQS ReceiveMessageRequest failed: ", err) + continue + } + + if output == nil || len(output.Messages) == 0 { + c.logger.Debug("no message received from SQS") + continue + } + + // process messages received from sqs, get logs from s3 and create events + c.processor(c.config.QueueURL, output.Messages, c.visibilityTimeout, c.s3, c.sqs) + } +} + +func (c *s3Collector) processor(queueURL string, messages []sqs.Message, visibilityTimeout int64, svcS3 s3iface.ClientAPI, svcSQS sqsiface.ClientAPI) { + var grp unison.MultiErrGroup + numMessages := len(messages) + c.logger.Debugf("Processing %v messages", numMessages) + + // process messages received from sqs + for i := range messages { + i := i + errC := make(chan error) + grp.Go(func() (err error) { + return c.processMessage(svcS3, messages[i], errC) + }) + grp.Go(func() (err error) { + return c.processorKeepAlive(svcSQS, messages[i], queueURL, visibilityTimeout, errC) + }) + } + grp.Wait() +} + +func (c *s3Collector) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, errC chan error) error { + s3Infos, err := c.handleSQSMessage(message) + if err != nil { + c.logger.Error(fmt.Errorf("handleSQSMessage failed: %w", err)) + return err + } + c.logger.Debugf("handleSQSMessage succeed and returned %v sets of S3 log info", len(s3Infos)) + + // read from s3 object and create event for each log line + err = c.handleS3Objects(svcS3, s3Infos, errC) + if err != nil { + c.logger.Error(fmt.Errorf("handleS3Objects failed: %w", err)) + return err + } + c.logger.Debugf("handleS3Objects succeed") + return nil +} + +func (c *s3Collector) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.Message, queueURL string, visibilityTimeout int64, errC chan error) error { + for { + select { + case <-c.cancellation.Done(): + return nil + case err := <-errC: + if err != nil { + if err == context.DeadlineExceeded { + c.logger.Info("Context deadline exceeded, updating visibility timeout") + } else { + c.logger.Warnf("Processing message failed '%w', updating visibility timeout", err) + } + + err := c.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) + if err != nil { + c.logger.Error(fmt.Errorf("SQS ChangeMessageVisibilityRequest failed: %w", err)) + } + c.logger.Infof("Message visibility timeout updated to %v", visibilityTimeout) + } else { + // When ACK done, message will be deleted. Or when message is + // not s3 ObjectCreated event related(handleSQSMessage function + // failed), it will be removed as well. + c.logger.Debug("Deleting message from SQS: ", *message.MessageId) + // only delete sqs message when errC is closed with no error + err := c.deleteMessage(queueURL, *message.ReceiptHandle, svcSQS) + if err != nil { + c.logger.Error(fmt.Errorf("deleteMessages failed: %w", err)) + } + } + return err + case <-time.After(time.Duration(visibilityTimeout/2) * time.Second): + c.logger.Warn("Half of the set visibilityTimeout passed, visibility timeout needs to be updated") + // If half of the set visibilityTimeout passed and this is + // still ongoing, then change visibility timeout. + err := c.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) + if err != nil { + c.logger.Error(fmt.Errorf("SQS ChangeMessageVisibilityRequest failed: %w", err)) + } + c.logger.Infof("Message visibility timeout updated to %v seconds", visibilityTimeout) + return err + } + } +} + +func (c *s3Collector) receiveMessage(svcSQS sqsiface.ClientAPI, visibilityTimeout int64) (*sqs.ReceiveMessageResponse, error) { + // receive messages from sqs + req := svcSQS.ReceiveMessageRequest( + &sqs.ReceiveMessageInput{ + QueueUrl: &c.config.QueueURL, + MessageAttributeNames: []string{"All"}, + MaxNumberOfMessages: awssdk.Int64(int64(maxNumberOfMessage)), + VisibilityTimeout: &visibilityTimeout, + WaitTimeSeconds: awssdk.Int64(int64(waitTimeSecond)), + }) + + // The Context will interrupt the request if the timeout expires. + sendCtx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + return req.Send(sendCtx) +} + +func (c *s3Collector) changeVisibilityTimeout(queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI, receiptHandle *string) error { + req := svcSQS.ChangeMessageVisibilityRequest(&sqs.ChangeMessageVisibilityInput{ + QueueUrl: &queueURL, + VisibilityTimeout: &visibilityTimeout, + ReceiptHandle: receiptHandle, + }) + + // The Context will interrupt the request if the timeout expires. + sendCtx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + _, err := req.Send(sendCtx) + return err +} + +func getRegionFromQueueURL(queueURL string) (string, error) { + // get region from queueURL + // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs + queueURLSplit := strings.Split(queueURL, ".") + if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { + return queueURLSplit[1], nil + } + return "", fmt.Errorf("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") +} + +// handle message +func (c *s3Collector) handleSQSMessage(m sqs.Message) ([]s3Info, error) { + msg := sqsMessage{} + err := json.Unmarshal([]byte(*m.Body), &msg) + if err != nil { + return nil, fmt.Errorf("json unmarshal sqs message body failed: %w", err) + } + + var s3Infos []s3Info + for _, record := range msg.Records { + if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { + return nil, fmt.Errorf("this SQS queue should be dedicated to s3 ObjectCreated event notifications") + } + // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" + filename, err := url.QueryUnescape(record.S3.object.Key) + if err != nil { + return nil, fmt.Errorf("url.QueryUnescape failed for '%s': %w", record.S3.object.Key, err) + } + + if len(c.config.FileSelectors) == 0 { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: c.config.ExpandEventListFromField, + }) + continue + } + + for _, fs := range c.config.FileSelectors { + if fs.Regex == nil { + continue + } + if fs.Regex.MatchString(filename) { + s3Infos = append(s3Infos, s3Info{ + region: record.AwsRegion, + name: record.S3.bucket.Name, + key: filename, + arn: record.S3.bucket.Arn, + expandEventListFromField: fs.ExpandEventListFromField, + }) + break + } + } + } + return s3Infos, nil +} + +func (c *s3Collector) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC chan error) error { + s3Ctx := &s3Context{ + refs: 1, + errC: errC, + } + defer s3Ctx.done() + + for _, info := range s3Infos { + c.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) + err := c.createEventsFromS3Info(svc, info, s3Ctx) + if err != nil { + c.logger.Error(fmt.Errorf("createEventsFromS3Info failed processing file from s3 bucket \"%s\" with name \"%s\": %w", info.name, info.key, err)) + s3Ctx.setError(err) + } + } + return nil +} + +func (c *s3Collector) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3Ctx *s3Context) error { + objectHash := s3ObjectHash(info) + + // Download the S3 object using GetObjectRequest. + s3GetObjectInput := &s3.GetObjectInput{ + Bucket: awssdk.String(info.name), + Key: awssdk.String(info.key), + } + req := svc.GetObjectRequest(s3GetObjectInput) + + // The Context will interrupt the request if the timeout expires. + ctx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + resp, err := req.Send(ctx) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + // If the SDK can determine the request or retry delay was canceled + // by a context the ErrCodeRequestCanceled error will be returned. + if awsErr.Code() == awssdk.ErrCodeRequestCanceled { + c.logger.Error(fmt.Errorf("s3 GetObjectRequest canceled for '%s' from S3 bucket '%s': %w", info.key, info.name, err)) + return err + } + + if awsErr.Code() == "NoSuchKey" { + c.logger.Warnf("Cannot find s3 file '%s' from S3 bucket '%s'", info.key, info.name) + return nil + } + } + return fmt.Errorf("s3 GetObjectRequest failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err) + } + + defer resp.Body.Close() + + reader := bufio.NewReader(resp.Body) + + isS3ObjGzipped, err := isStreamGzipped(reader) + if err != nil { + c.logger.Error(fmt.Errorf("could not determine if S3 object is gzipped: %w", err)) + return err + } + + if isS3ObjGzipped { + gzipReader, err := gzip.NewReader(reader) + if err != nil { + c.logger.Error(fmt.Errorf("gzip.NewReader failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err)) + return err + } + reader = bufio.NewReader(gzipReader) + gzipReader.Close() + } + + // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config + if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { + decoder := json.NewDecoder(reader) + err := c.decodeJSON(decoder, objectHash, info, s3Ctx) + if err != nil { + c.logger.Error(fmt.Errorf("decodeJSONWithKey failed for '%s' from S3 bucket '%s': %w", info.key, info.name, err)) + return err + } + return nil + } + + // handle s3 objects that are not json content-type + offset := 0 + for { + log, err := readStringAndTrimDelimiter(reader) + if err == io.EOF { + // create event for last line + offset += len([]byte(log)) + event := createEvent(log, offset, info, objectHash, s3Ctx) + err = c.forwardEvent(event) + if err != nil { + c.logger.Error(fmt.Errorf("forwardEvent failed: %w", err)) + return err + } + return nil + } else if err != nil { + c.logger.Error(fmt.Errorf("readStringAndTrimDelimiter failed: %w", err)) + return err + } + + if log == "" { + break + } + + // create event per log line + offset += len([]byte(log)) + event := createEvent(log, offset, info, objectHash, s3Ctx) + err = c.forwardEvent(event) + if err != nil { + c.logger.Error(fmt.Errorf("forwardEvent failed: %w", err)) + return err + } + } + return nil +} + +func (c *s3Collector) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3Info, s3Ctx *s3Context) error { + offset := 0 + for { + var jsonFields interface{} + err := decoder.Decode(&jsonFields) + if jsonFields == nil { + return nil + } + + if err == io.EOF { + offsetNew, err := c.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + if err != nil { + return err + } + offset = offsetNew + } else if err != nil { + // decode json failed, skip this log file + err = fmt.Errorf("decode json failed for '%s' from S3 bucket '%s', skipping this file: %w", s3Info.key, s3Info.name, err) + c.logger.Warn(err) + return nil + } + + offset, err = c.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) + if err != nil { + return err + } + } +} + +func (c *s3Collector) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { + switch f := jsonFields.(type) { + case map[string][]interface{}: + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] + if !ok { + err := fmt.Errorf("key '%s' not found", s3Info.expandEventListFromField) + c.logger.Error(err) + return offset, err + } + for _, v := range textValues { + offset, err := c.convertJSONToEvent(v, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + } + return offset, nil + } + case map[string]interface{}: + if s3Info.expandEventListFromField != "" { + textValues, ok := f[s3Info.expandEventListFromField] + if !ok { + err := fmt.Errorf("key '%s' not found", s3Info.expandEventListFromField) + c.logger.Error(err) + return offset, err + } + + valuesConverted := textValues.([]interface{}) + for _, textValue := range valuesConverted { + offsetNew, err := c.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + offset = offsetNew + } + return offset, nil + } + + offset, err := c.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) + if err != nil { + err = fmt.Errorf("convertJSONToEvent failed for '%s' from S3 bucket '%s': %w", s3Info.key, s3Info.name, err) + c.logger.Error(err) + return offset, err + } + return offset, nil + } + return offset, nil +} + +func (c *s3Collector) convertJSONToEvent(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { + vJSON, _ := json.Marshal(jsonFields) + logOriginal := string(vJSON) + log := trimLogDelimiter(logOriginal) + offset += len([]byte(log)) + event := createEvent(log, offset, s3Info, objectHash, s3Ctx) + + err := c.forwardEvent(event) + if err != nil { + err = fmt.Errorf("forwardEvent failed: %w", err) + c.logger.Error(err) + return offset, err + } + return offset, nil +} + +func (c *s3Collector) forwardEvent(event beat.Event) error { + c.publisher.Publish(event) + return c.cancellation.Err() +} + +func (c *s3Collector) deleteMessage(queueURL string, messagesReceiptHandle string, svcSQS sqsiface.ClientAPI) error { + deleteMessageInput := &sqs.DeleteMessageInput{ + QueueUrl: awssdk.String(queueURL), + ReceiptHandle: awssdk.String(messagesReceiptHandle), + } + + req := svcSQS.DeleteMessageRequest(deleteMessageInput) + + // The Context will interrupt the request if the timeout expires. + ctx, cancelFn := context.WithTimeout(c.cancellation, c.config.APITimeout) + defer cancelFn() + + _, err := req.Send(ctx) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { + return nil + } + return fmt.Errorf("SQS DeleteMessageRequest failed: %w", err) + } + return nil +} + +func trimLogDelimiter(log string) string { + return strings.TrimSuffix(log, "\n") +} + +func readStringAndTrimDelimiter(reader *bufio.Reader) (string, error) { + logOriginal, err := reader.ReadString('\n') + if err != nil { + return logOriginal, err + } + return trimLogDelimiter(logOriginal), nil +} + +func createEvent(log string, offset int, info s3Info, objectHash string, s3Ctx *s3Context) beat.Event { + s3Ctx.Inc() + + event := beat.Event{ + Timestamp: time.Now().UTC(), + Fields: common.MapStr{ + "message": log, + "log": common.MapStr{ + "offset": int64(offset), + "file": common.MapStr{ + "path": constructObjectURL(info), + }, + }, + "aws": common.MapStr{ + "s3": common.MapStr{ + "bucket": common.MapStr{ + "name": info.name, + "arn": info.arn}, + "object.key": info.key, + }, + }, + "cloud": common.MapStr{ + "provider": "aws", + "region": info.region, + }, + }, + Private: s3Ctx, + } + event.SetID(objectHash + "-" + fmt.Sprintf("%012d", offset)) + + return event +} + +func constructObjectURL(info s3Info) string { + return "https://" + info.name + ".s3-" + info.region + ".amazonaws.com/" + info.key +} + +// s3ObjectHash returns a short sha256 hash of the bucket arn + object key name. +func s3ObjectHash(s3Info s3Info) string { + h := sha256.New() + h.Write([]byte(s3Info.arn + s3Info.key)) + prefix := hex.EncodeToString(h.Sum(nil)) + return prefix[:10] +} + +func (c *s3Context) setError(err error) { + // only care about the last error for now + // TODO: add "Typed" error to error for context + c.mux.Lock() + defer c.mux.Unlock() + c.err = err +} + +func (c *s3Context) done() { + c.mux.Lock() + defer c.mux.Unlock() + c.refs-- + if c.refs == 0 { + c.errC <- c.err + close(c.errC) + } +} + +func (c *s3Context) Inc() { + c.mux.Lock() + defer c.mux.Unlock() + c.refs++ +} + +// isStreamGzipped determines whether the given stream of bytes (encapsulated in a buffered reader) +// represents gzipped content or not. A buffered reader is used so the function can peek into the byte +// stream without consuming it. This makes it convenient for code executed after this function call +// to consume the stream if it wants. +func isStreamGzipped(r *bufio.Reader) (bool, error) { + // Why 512? See https://godoc.org/net/http#DetectContentType + buf, err := r.Peek(512) + if err != nil && err != io.EOF { + return false, err + } + + switch http.DetectContentType(buf) { + case "application/x-gzip", "application/zip": + return true, nil + default: + return false, nil + } +} diff --git a/x-pack/filebeat/input/s3/input_test.go b/x-pack/filebeat/input/s3/collector_test.go similarity index 97% rename from x-pack/filebeat/input/s3/input_test.go rename to x-pack/filebeat/input/s3/collector_test.go index d1fab05cb3c..510f94d40d5 100644 --- a/x-pack/filebeat/input/s3/input_test.go +++ b/x-pack/filebeat/input/s3/collector_test.go @@ -120,7 +120,7 @@ func TestHandleMessage(t *testing.T) { }, } - p := &s3Input{context: &channelContext{}} + p := &s3Collector{config: &config{}} for _, c := range casesPositive { t.Run(c.title, func(t *testing.T) { s3Info, err := p.handleSQSMessage(c.message) @@ -165,7 +165,8 @@ func TestHandleMessage(t *testing.T) { } func TestNewS3BucketReader(t *testing.T) { - p := &s3Input{context: &channelContext{}} + config := defaultConfig() + p := &s3Collector{cancellation: context.TODO(), config: &config} s3GetObjectInput := &s3.GetObjectInput{ Bucket: awssdk.String(info.name), Key: awssdk.String(info.key), @@ -174,7 +175,7 @@ func TestNewS3BucketReader(t *testing.T) { // The Context will interrupt the request if the timeout expires. var cancelFn func() - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) + ctx, cancelFn := context.WithTimeout(p.cancellation, p.config.APITimeout) defer cancelFn() resp, err := req.Send(ctx) @@ -201,7 +202,8 @@ func TestNewS3BucketReader(t *testing.T) { } func TestCreateEvent(t *testing.T) { - p := &s3Input{context: &channelContext{}} + config := defaultConfig() + p := &s3Collector{cancellation: context.TODO(), config: &config} errC := make(chan error) s3Context := &s3Context{ refs: 1, @@ -225,7 +227,7 @@ func TestCreateEvent(t *testing.T) { // The Context will interrupt the request if the timeout expires. var cancelFn func() - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) + ctx, cancelFn := context.WithTimeout(p.cancellation, p.config.APITimeout) defer cancelFn() resp, err := req.Send(ctx) diff --git a/x-pack/filebeat/input/s3/config.go b/x-pack/filebeat/input/s3/config.go index f9780d82277..cc3c5318289 100644 --- a/x-pack/filebeat/input/s3/config.go +++ b/x-pack/filebeat/input/s3/config.go @@ -9,18 +9,17 @@ import ( "regexp" "time" - "github.com/elastic/beats/v7/filebeat/harvester" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" ) type config struct { - harvester.ForwarderConfig `config:",inline"` - QueueURL string `config:"queue_url" validate:"nonzero,required"` - VisibilityTimeout time.Duration `config:"visibility_timeout"` - AwsConfig awscommon.ConfigAWS `config:",inline"` - ExpandEventListFromField string `config:"expand_event_list_from_field"` - APITimeout time.Duration `config:"api_timeout"` - FileSelectors []FileSelectorCfg `config:"file_selectors"` + QueueURL string `config:"queue_url" validate:"nonzero,required"` + VisibilityTimeout time.Duration `config:"visibility_timeout"` + FipsEnabled bool `config:"fips_enabled"` + AwsConfig awscommon.ConfigAWS `config:",inline"` + ExpandEventListFromField string `config:"expand_event_list_from_field"` + APITimeout time.Duration `config:"api_timeout"` + FileSelectors []FileSelectorCfg `config:"file_selectors"` } // FileSelectorCfg defines type and configuration of FileSelectors @@ -32,11 +31,9 @@ type FileSelectorCfg struct { func defaultConfig() config { return config{ - ForwarderConfig: harvester.ForwarderConfig{ - Type: "s3", - }, VisibilityTimeout: 300 * time.Second, APITimeout: 120 * time.Second, + FipsEnabled: false, } } diff --git a/x-pack/filebeat/input/s3/input.go b/x-pack/filebeat/input/s3/input.go index 83dc48428ee..d76e5b8b728 100644 --- a/x-pack/filebeat/input/s3/input.go +++ b/x-pack/filebeat/input/s3/input.go @@ -5,748 +5,127 @@ package s3 import ( - "bufio" - "compress/gzip" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" "fmt" - "io" - "net/http" - "net/url" - "strings" - "sync" - "time" - awssdk "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/aws/awserr" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" "github.com/aws/aws-sdk-go-v2/service/sqs" - "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/acker" - "github.com/elastic/beats/v7/libbeat/common/cfgwarn" - "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/feature" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/go-concert/ctxtool" ) const inputName = "s3" -var ( - // The maximum number of messages to return. Amazon SQS never returns more messages - // than this value (however, fewer messages might be returned). - maxNumberOfMessage int64 = 10 - - // The duration (in seconds) for which the call waits for a message to arrive - // in the queue before returning. If a message is available, the call returns - // sooner than WaitTimeSeconds. If no messages are available and the wait time - // expires, the call returns successfully with an empty list of messages. - waitTimeSecond int64 = 10 - - errOutletClosed = errors.New("input outlet closed") -) - -func init() { - err := input.Register(inputName, NewInput) - if err != nil { - panic(err) +func Plugin() v2.Plugin { + return v2.Plugin{ + Name: inputName, + Stability: feature.Beta, + Deprecated: false, + Info: "Collect logs from s3", + Manager: v2.ConfigureWith(configure), } } -// s3Input is a input for s3 -type s3Input struct { - outlet channel.Outleter // Output of received s3 logs. - config config - awsConfig awssdk.Config - logger *logp.Logger - close chan struct{} - workerOnce sync.Once // Guarantees that the worker goroutine is only started once. - context *channelContext - workerWg sync.WaitGroup // Waits on s3 worker goroutine. - stopOnce sync.Once -} - -type s3Info struct { - name string - key string - region string - arn string - expandEventListFromField string -} - -type bucket struct { - Name string `json:"name"` - Arn string `json:"arn"` -} - -type object struct { - Key string `json:"key"` -} - -type s3BucketObject struct { - bucket `json:"bucket"` - object `json:"object"` -} - -type sqsMessage struct { - Records []struct { - EventSource string `json:"eventSource"` - AwsRegion string `json:"awsRegion"` - EventName string `json:"eventName"` - S3 s3BucketObject `json:"s3"` - } `json:"Records"` -} - -type s3Context struct { - mux sync.Mutex - refs int - err error // first error witnessed or multi error - errC chan error -} - -// channelContext implements context.Context by wrapping a channel -type channelContext struct { - done <-chan struct{} -} - -func (c *channelContext) Deadline() (time.Time, bool) { return time.Time{}, false } -func (c *channelContext) Done() <-chan struct{} { return c.done } -func (c *channelContext) Err() error { - select { - case <-c.done: - return context.Canceled - default: - return nil - } -} -func (c *channelContext) Value(key interface{}) interface{} { return nil } - -// NewInput creates a new s3 input -func NewInput(cfg *common.Config, connector channel.Connector, context input.Context) (input.Input, error) { - cfgwarn.Beta("s3 input type is used") - logger := logp.NewLogger(inputName) - +func configure(cfg *common.Config) (v2.Input, error) { config := defaultConfig() if err := cfg.Unpack(&config); err != nil { - return nil, errors.Wrap(err, "failed unpacking config") - } - - out, err := connector.ConnectWith(cfg, beat.ClientConfig{ - ACKHandler: acker.ConnectionOnly( - acker.EventPrivateReporter(func(_ int, privates []interface{}) { - for _, private := range privates { - if s3Context, ok := private.(*s3Context); ok { - s3Context.done() - } - } - }), - ), - }) - if err != nil { return nil, err } - awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) - if err != nil { - return nil, errors.Wrap(err, "getAWSCredentials failed") - } - - closeChannel := make(chan struct{}) - p := &s3Input{ - outlet: out, - config: config, - awsConfig: awsConfig, - logger: logger, - close: closeChannel, - context: &channelContext{closeChannel}, - } - return p, nil -} - -// Run runs the input -func (p *s3Input) Run() { - p.workerOnce.Do(func() { - visibilityTimeout := int64(p.config.VisibilityTimeout.Seconds()) - p.logger.Infof("visibility timeout is set to %v seconds", visibilityTimeout) - p.logger.Infof("aws api timeout is set to %v", p.config.APITimeout) - - regionName, err := getRegionFromQueueURL(p.config.QueueURL) - if err != nil { - p.logger.Errorf("failed to get region name from queueURL: %v", p.config.QueueURL) - } - - awsConfig := p.awsConfig.Copy() - awsConfig.Region = regionName - - svcSQS := sqs.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)) - svcS3 := s3.New(awscommon.EnrichAWSConfigWithEndpoint(p.config.AwsConfig.Endpoint, "s3", regionName, awsConfig)) - - p.workerWg.Add(1) - go p.run(svcSQS, svcS3, visibilityTimeout) - }) -} - -func (p *s3Input) run(svcSQS sqsiface.ClientAPI, svcS3 s3iface.ClientAPI, visibilityTimeout int64) { - defer p.workerWg.Done() - defer p.logger.Infof("s3 input worker for '%v' has stopped.", p.config.QueueURL) - - p.logger.Infof("s3 input worker has started. with queueURL: %v", p.config.QueueURL) - for p.context.Err() == nil { - // receive messages from sqs - output, err := p.receiveMessage(svcSQS, visibilityTimeout) - if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { - continue - } - p.logger.Error("SQS ReceiveMessageRequest failed: ", err) - time.Sleep(time.Duration(waitTimeSecond) * time.Second) - continue - } - - if output == nil || len(output.Messages) == 0 { - p.logger.Debug("no message received from SQS:", p.config.QueueURL) - continue - } - - // process messages received from sqs, get logs from s3 and create events - p.processor(p.config.QueueURL, output.Messages, visibilityTimeout, svcS3, svcSQS) - } -} - -// Stop stops the s3 input -func (p *s3Input) Stop() { - p.stopOnce.Do(func() { - defer p.outlet.Close() - close(p.close) - p.logger.Info("Stopping s3 input") - }) -} - -// Wait stops the s3 input. -func (p *s3Input) Wait() { - p.Stop() - p.workerWg.Wait() -} - -func (p *s3Input) processor(queueURL string, messages []sqs.Message, visibilityTimeout int64, svcS3 s3iface.ClientAPI, svcSQS sqsiface.ClientAPI) { - var wg sync.WaitGroup - numMessages := len(messages) - p.logger.Debugf("Processing %v messages", numMessages) - wg.Add(numMessages * 2) - - // process messages received from sqs - for i := range messages { - errC := make(chan error) - go p.processMessage(svcS3, messages[i], &wg, errC) - go p.processorKeepAlive(svcSQS, messages[i], queueURL, visibilityTimeout, &wg, errC) - } - wg.Wait() + return newInput(config) } -func (p *s3Input) processMessage(svcS3 s3iface.ClientAPI, message sqs.Message, wg *sync.WaitGroup, errC chan error) { - defer wg.Done() - - s3Infos, err := p.handleSQSMessage(message) - if err != nil { - p.logger.Error(errors.Wrap(err, "handleSQSMessage failed")) - return - } - p.logger.Debugf("handleSQSMessage succeed and returned %v sets of S3 log info", len(s3Infos)) - - // read from s3 object and create event for each log line - err = p.handleS3Objects(svcS3, s3Infos, errC) - if err != nil { - err = errors.Wrap(err, "handleS3Objects failed") - p.logger.Error(err) - return - } - p.logger.Debugf("handleS3Objects succeed") -} - -func (p *s3Input) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.Message, queueURL string, visibilityTimeout int64, wg *sync.WaitGroup, errC chan error) { - defer wg.Done() - for { - select { - case <-p.close: - return - case err := <-errC: - if err != nil { - p.logger.Warn("Processing message failed, updating visibility timeout") - err := p.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) - if err != nil { - p.logger.Error(errors.Wrap(err, "SQS ChangeMessageVisibilityRequest failed")) - } - p.logger.Infof("Message visibility timeout updated to %v", visibilityTimeout) - } else { - // When ACK done, message will be deleted. Or when message is - // not s3 ObjectCreated event related(handleSQSMessage function - // failed), it will be removed as well. - p.logger.Debug("Deleting message from SQS: ", *message.MessageId) - // only delete sqs message when errC is closed with no error - err := p.deleteMessage(queueURL, *message.ReceiptHandle, svcSQS) - if err != nil { - p.logger.Error(errors.Wrap(err, "deleteMessages failed")) - } - } - return - case <-time.After(time.Duration(visibilityTimeout/2) * time.Second): - p.logger.Warn("Half of the set visibilityTimeout passed, visibility timeout needs to be updated") - // If half of the set visibilityTimeout passed and this is - // still ongoing, then change visibility timeout. - err := p.changeVisibilityTimeout(queueURL, visibilityTimeout, svcSQS, message.ReceiptHandle) - if err != nil { - p.logger.Error(errors.Wrap(err, "SQS ChangeMessageVisibilityRequest failed")) - } - p.logger.Infof("Message visibility timeout updated to %v seconds", visibilityTimeout) - } - } -} - -func (p *s3Input) receiveMessage(svcSQS sqsiface.ClientAPI, visibilityTimeout int64) (*sqs.ReceiveMessageResponse, error) { - // receive messages from sqs - req := svcSQS.ReceiveMessageRequest( - &sqs.ReceiveMessageInput{ - QueueUrl: &p.config.QueueURL, - MessageAttributeNames: []string{"All"}, - MaxNumberOfMessages: &maxNumberOfMessage, - VisibilityTimeout: &visibilityTimeout, - WaitTimeSeconds: &waitTimeSecond, - }) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - return req.Send(ctx) +// s3Input is a input for s3 +type s3Input struct { + config config } -func (p *s3Input) changeVisibilityTimeout(queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI, receiptHandle *string) error { - req := svcSQS.ChangeMessageVisibilityRequest(&sqs.ChangeMessageVisibilityInput{ - QueueUrl: &queueURL, - VisibilityTimeout: &visibilityTimeout, - ReceiptHandle: receiptHandle, - }) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - _, err := req.Send(ctx) - return err +func newInput(config config) (*s3Input, error) { + return &s3Input{config: config}, nil } -func getRegionFromQueueURL(queueURL string) (string, error) { - // get region from queueURL - // Example: https://sqs.us-east-1.amazonaws.com/627959692251/test-s3-logs - queueURLSplit := strings.Split(queueURL, ".") - if queueURLSplit[0] == "https://sqs" && queueURLSplit[2] == "amazonaws" { - return queueURLSplit[1], nil - } - return "", errors.New("queueURL is not in format: https://sqs.{REGION_ENDPOINT}.amazonaws.com/{ACCOUNT_NUMBER}/{QUEUE_NAME}") -} +func (in *s3Input) Name() string { return inputName } -// handle message -func (p *s3Input) handleSQSMessage(m sqs.Message) ([]s3Info, error) { - msg := sqsMessage{} - err := json.Unmarshal([]byte(*m.Body), &msg) +func (in *s3Input) Test(ctx v2.TestContext) error { + _, err := awscommon.GetAWSCredentials(in.config.AwsConfig) if err != nil { - return nil, errors.Wrap(err, "json unmarshal sqs message body failed") - } - - var s3Infos []s3Info - for _, record := range msg.Records { - if record.EventSource != "aws:s3" || !strings.HasPrefix(record.EventName, "ObjectCreated:") { - return nil, errors.New("this SQS queue should be dedicated to s3 ObjectCreated event notifications") - } - // Unescape substrings from s3 log name. For example, convert "%3D" back to "=" - filename, err := url.QueryUnescape(record.S3.object.Key) - if err != nil { - return nil, errors.Wrapf(err, "url.QueryUnescape failed for '%s'", record.S3.object.Key) - } - - if len(p.config.FileSelectors) == 0 { - s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, - expandEventListFromField: p.config.ExpandEventListFromField, - }) - continue - } - - for _, fs := range p.config.FileSelectors { - if fs.Regex == nil { - continue - } - if fs.Regex.MatchString(filename) { - s3Infos = append(s3Infos, s3Info{ - region: record.AwsRegion, - name: record.S3.bucket.Name, - key: filename, - arn: record.S3.bucket.Arn, - expandEventListFromField: fs.ExpandEventListFromField, - }) - break - } - } - } - return s3Infos, nil -} - -func (p *s3Input) handleS3Objects(svc s3iface.ClientAPI, s3Infos []s3Info, errC chan error) error { - s3Ctx := &s3Context{ - refs: 1, - errC: errC, - } - defer s3Ctx.done() - - for _, info := range s3Infos { - p.logger.Debugf("Processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) - err := p.createEventsFromS3Info(svc, info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "createEventsFromS3Info failed processing file from s3 bucket \"%s\" with name \"%s\"", info.name, info.key) - p.logger.Error(err) - s3Ctx.setError(err) - } + return fmt.Errorf("getAWSCredentials failed: %w", err) } return nil } -func (p *s3Input) createEventsFromS3Info(svc s3iface.ClientAPI, info s3Info, s3Ctx *s3Context) error { - objectHash := s3ObjectHash(info) - - // Download the S3 object using GetObjectRequest. - s3GetObjectInput := &s3.GetObjectInput{ - Bucket: awssdk.String(info.name), - Key: awssdk.String(info.key), - } - req := svc.GetObjectRequest(s3GetObjectInput) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - resp, err := req.Send(ctx) +func (in *s3Input) Run(ctx v2.Context, pipeline beat.Pipeline) error { + collector, err := in.createCollector(ctx, pipeline) if err != nil { - if awsErr, ok := err.(awserr.Error); ok { - // If the SDK can determine the request or retry delay was canceled - // by a context the ErrCodeRequestCanceled error will be returned. - if awsErr.Code() == awssdk.ErrCodeRequestCanceled { - err = errors.Wrapf(err, "S3 GetObjectRequest canceled for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - - if awsErr.Code() == "NoSuchKey" { - p.logger.Warnf("Cannot find s3 file '%s' from S3 bucket '%s'", info.key, info.name) - return nil - } - } - return errors.Wrapf(err, "S3 GetObjectRequest failed for '%s' from S3 bucket '%s'", info.key, info.name) - } - - defer resp.Body.Close() - - reader := bufio.NewReader(resp.Body) - - isS3ObjGzipped, err := isStreamGzipped(reader) - if err != nil { - err = errors.Wrap(err, "could not determine if S3 object is gzipped") - p.logger.Error(err) return err } - if isS3ObjGzipped { - gzipReader, err := gzip.NewReader(reader) - if err != nil { - err = errors.Wrapf(err, "gzip.NewReader failed for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - reader = bufio.NewReader(gzipReader) - gzipReader.Close() - } - - // Decode JSON documents when content-type is "application/json" or expand_event_list_from_field is given in config - if resp.ContentType != nil && *resp.ContentType == "application/json" || info.expandEventListFromField != "" { - decoder := json.NewDecoder(reader) - err := p.decodeJSON(decoder, objectHash, info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "decodeJSONWithKey failed for '%s' from S3 bucket '%s'", info.key, info.name) - p.logger.Error(err) - return err - } - return nil - } - - // handle s3 objects that are not json content-type - offset := 0 - for { - log, err := readStringAndTrimDelimiter(reader) - if err == io.EOF { - // create event for last line - offset += len([]byte(log)) - event := createEvent(log, offset, info, objectHash, s3Ctx) - err = p.forwardEvent(event) - if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return err - } - return nil - } else if err != nil { - err = errors.Wrap(err, "readStringAndTrimDelimiter failed") - p.logger.Error(err) - return err - } - - if log == "" { - break - } - - // create event per log line - offset += len([]byte(log)) - event := createEvent(log, offset, info, objectHash, s3Ctx) - err = p.forwardEvent(event) - if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return err - } - } - return nil -} - -func (p *s3Input) decodeJSON(decoder *json.Decoder, objectHash string, s3Info s3Info, s3Ctx *s3Context) error { - offset := 0 - for { - var jsonFields interface{} - err := decoder.Decode(&jsonFields) - if jsonFields == nil { - return nil - } - - if err == io.EOF { - offset, err = p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) - if err != nil { - return err - } - } else if err != nil { - // decode json failed, skip this log file - err = errors.Wrapf(err, "decode json failed for '%s' from S3 bucket '%s', skipping this file", s3Info.key, s3Info.name) - p.logger.Warn(err) - return nil - } - - offsetNew, err := p.jsonFieldsType(jsonFields, offset, objectHash, s3Info, s3Ctx) - if err != nil { - return err - } - offset = offsetNew - } -} - -func (p *s3Input) jsonFieldsType(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { - switch f := jsonFields.(type) { - case map[string][]interface{}: - if s3Info.expandEventListFromField != "" { - textValues, ok := f[s3Info.expandEventListFromField] - if !ok { - err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) - p.logger.Error(err) - return offset, err - } - for _, v := range textValues { - offset, err := p.convertJSONToEvent(v, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - } - return offset, nil - } - case map[string]interface{}: - if s3Info.expandEventListFromField != "" { - textValues, ok := f[s3Info.expandEventListFromField] - if !ok { - err := errors.Errorf("key '%s' not found", s3Info.expandEventListFromField) - p.logger.Error(err) - return offset, err - } - - valuesConverted := textValues.([]interface{}) - for _, textValue := range valuesConverted { - offsetNew, err := p.convertJSONToEvent(textValue, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - offset = offsetNew - } - return offset, nil - } - - offset, err := p.convertJSONToEvent(f, offset, objectHash, s3Info, s3Ctx) - if err != nil { - err = errors.Wrapf(err, "convertJSONToEvent failed for '%s' from S3 bucket '%s'", s3Info.key, s3Info.name) - p.logger.Error(err) - return offset, err - } - return offset, nil - } - return offset, nil + defer collector.publisher.Close() + collector.run() + return ctx.Cancelation.Err() } -func (p *s3Input) convertJSONToEvent(jsonFields interface{}, offset int, objectHash string, s3Info s3Info, s3Ctx *s3Context) (int, error) { - vJSON, err := json.Marshal(jsonFields) - logOriginal := string(vJSON) - log := trimLogDelimiter(logOriginal) - offset += len([]byte(log)) - event := createEvent(log, offset, s3Info, objectHash, s3Ctx) +func (in *s3Input) createCollector(ctx v2.Context, pipeline beat.Pipeline) (*s3Collector, error) { + log := ctx.Logger.With("queue_url", in.config.QueueURL) - err = p.forwardEvent(event) + client, err := pipeline.ConnectWith(beat.ClientConfig{ + CloseRef: ctx.Cancelation, + ACKHandler: newACKHandler(), + }) if err != nil { - err = errors.Wrap(err, "forwardEvent failed") - p.logger.Error(err) - return offset, err - } - return offset, nil -} - -func (p *s3Input) forwardEvent(event beat.Event) error { - ok := p.outlet.OnEvent(event) - if !ok { - return errOutletClosed - } - return nil -} - -func (p *s3Input) deleteMessage(queueURL string, messagesReceiptHandle string, svcSQS sqsiface.ClientAPI) error { - deleteMessageInput := &sqs.DeleteMessageInput{ - QueueUrl: awssdk.String(queueURL), - ReceiptHandle: awssdk.String(messagesReceiptHandle), + return nil, err } - req := svcSQS.DeleteMessageRequest(deleteMessageInput) - - // The Context will interrupt the request if the timeout expires. - ctx, cancelFn := context.WithTimeout(p.context, p.config.APITimeout) - defer cancelFn() - - _, err := req.Send(ctx) + regionName, err := getRegionFromQueueURL(in.config.QueueURL) if err != nil { - if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == awssdk.ErrCodeRequestCanceled { - return nil - } - return errors.Wrapf(err, "SQS DeleteMessageRequest failed in queue %s", queueURL) + err := fmt.Errorf("getRegionFromQueueURL failed: %w", err) + log.Error(err) + return nil, err + } else { + log = log.With("region", regionName) } - return nil -} -func trimLogDelimiter(log string) string { - return strings.TrimSuffix(log, "\n") -} - -func readStringAndTrimDelimiter(reader *bufio.Reader) (string, error) { - logOriginal, err := reader.ReadString('\n') + awsConfig, err := awscommon.GetAWSCredentials(in.config.AwsConfig) if err != nil { - return logOriginal, err + return nil, fmt.Errorf("getAWSCredentials failed: %w", err) } - return trimLogDelimiter(logOriginal), nil -} + awsConfig.Region = regionName -func createEvent(log string, offset int, info s3Info, objectHash string, s3Ctx *s3Context) beat.Event { - s3Ctx.Inc() + visibilityTimeout := int64(in.config.VisibilityTimeout.Seconds()) + log.Infof("visibility timeout is set to %v seconds", visibilityTimeout) + log.Infof("aws api timeout is set to %v", in.config.APITimeout) - event := beat.Event{ - Timestamp: time.Now().UTC(), - Fields: common.MapStr{ - "message": log, - "log": common.MapStr{ - "offset": int64(offset), - "file.path": constructObjectURL(info), - }, - "aws": common.MapStr{ - "s3": common.MapStr{ - "bucket": common.MapStr{ - "name": info.name, - "arn": info.arn}, - "object.key": info.key, - }, - }, - "cloud": common.MapStr{ - "provider": "aws", - "region": info.region, - }, - }, - Private: s3Ctx, + s3Servicename := "s3" + if in.config.FipsEnabled { + s3Servicename = "s3-fips" } - event.SetID(objectHash + "-" + fmt.Sprintf("%012d", offset)) - - return event -} - -func constructObjectURL(info s3Info) string { - return "https://" + info.name + ".s3-" + info.region + ".amazonaws.com/" + info.key -} - -// s3ObjectHash returns a short sha256 hash of the bucket arn + object key name. -func s3ObjectHash(s3Info s3Info) string { - h := sha256.New() - h.Write([]byte(s3Info.arn + s3Info.key)) - prefix := hex.EncodeToString(h.Sum(nil)) - return prefix[:10] -} - -func (c *s3Context) setError(err error) { - // only care about the last error for now - // TODO: add "Typed" error to error for context - c.mux.Lock() - defer c.mux.Unlock() - c.err = err -} -func (c *s3Context) done() { - c.mux.Lock() - defer c.mux.Unlock() - c.refs-- - if c.refs == 0 { - c.errC <- c.err - close(c.errC) - } -} + log.Debug("s3 service name = ", s3Servicename) -func (c *s3Context) Inc() { - c.mux.Lock() - defer c.mux.Unlock() - c.refs++ + return &s3Collector{ + cancellation: ctxtool.FromCanceller(ctx.Cancelation), + logger: log, + config: &in.config, + publisher: client, + visibilityTimeout: visibilityTimeout, + sqs: sqs.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "sqs", regionName, awsConfig)), + s3: s3.New(awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, s3Servicename, regionName, awsConfig)), + }, nil } -// isStreamGzipped determines whether the given stream of bytes (encapsulated in a buffered reader) -// represents gzipped content or not. A buffered reader is used so the function can peek into the byte -// stream without consuming it. This makes it convenient for code executed after this function call -// to consume the stream if it wants. -func isStreamGzipped(r *bufio.Reader) (bool, error) { - // Why 512? See https://godoc.org/net/http#DetectContentType - buf, err := r.Peek(512) - if err != nil && err != io.EOF { - return false, err - } - - switch http.DetectContentType(buf) { - case "application/x-gzip", "application/zip": - return true, nil - default: - return false, nil - } +func newACKHandler() beat.ACKer { + return acker.ConnectionOnly( + acker.EventPrivateReporter(func(_ int, privates []interface{}) { + for _, private := range privates { + if s3Context, ok := private.(*s3Context); ok { + s3Context.done() + } + } + }), + ) } diff --git a/x-pack/filebeat/input/s3/s3_integration_test.go b/x-pack/filebeat/input/s3/s3_integration_test.go index 1d6a400a7f1..cadb5da035b 100644 --- a/x-pack/filebeat/input/s3/s3_integration_test.go +++ b/x-pack/filebeat/input/s3/s3_integration_test.go @@ -16,21 +16,20 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" - - "github.com/aws/aws-sdk-go-v2/aws" awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/s3iface" "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/sqsiface" + "github.com/stretchr/testify/assert" - "github.com/elastic/beats/v7/filebeat/channel" - "github.com/elastic/beats/v7/filebeat/input" + v2 "github.com/elastic/beats/v7/filebeat/input/v2" "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" + pubtest "github.com/elastic/beats/v7/libbeat/publisher/testing" "github.com/elastic/beats/v7/libbeat/tests/resources" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" + "github.com/elastic/go-concert/unison" ) const ( @@ -77,228 +76,99 @@ func getConfigForTest(t *testing.T) config { return config } -func uploadSampleLogFile(t *testing.T, bucketName string, svcS3 s3iface.ClientAPI) { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - file, err := os.Open(filePath) - if err != nil { - t.Fatalf("Failed to open file %v", filePath) - } - defer file.Close() - - s3PutObjectInput := s3.PutObjectInput{ - Bucket: aws.String(bucketName), - Key: aws.String(filepath.Base(filePath)), - Body: file, - } - req := svcS3.PutObjectRequest(&s3PutObjectInput) - output, err := req.Send(ctx) - if err != nil { - t.Fatalf("failed to put object into s3 bucket: %v", output) - } -} - -func collectOldMessages(t *testing.T, queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI) []string { - t.Helper() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - // receive messages from sqs - req := svcSQS.ReceiveMessageRequest( - &sqs.ReceiveMessageInput{ - QueueUrl: &queueURL, - MessageAttributeNames: []string{"All"}, - MaxNumberOfMessages: &maxNumberOfMessage, - VisibilityTimeout: &visibilityTimeout, - WaitTimeSeconds: &waitTimeSecond, - }) - - output, err := req.Send(ctx) - if err != nil { - t.Fatalf("failed to receive message from SQS: %v", output) - } - - var oldMessageHandles []string - for _, message := range output.Messages { - oldMessageHandles = append(oldMessageHandles, *message.ReceiptHandle) - } - - return oldMessageHandles -} - -func (input *s3Input) deleteAllMessages(t *testing.T, awsConfig awssdk.Config, queueURL string, visibilityTimeout int64, svcSQS sqsiface.ClientAPI) error { - var messageReceiptHandles []string - init := true - for init || len(messageReceiptHandles) > 0 { - init = false - messageReceiptHandles = collectOldMessages(t, queueURL, visibilityTimeout, svcSQS) - for _, receiptHandle := range messageReceiptHandles { - err := input.deleteMessage(queueURL, receiptHandle, svcSQS) - if err != nil { - return err - } - } - } - return nil -} - func defaultTestConfig() *common.Config { return common.MustNewConfigFrom(map[string]interface{}{ "queue_url": os.Getenv("QUEUE_URL"), }) } -func runTest(t *testing.T, cfg *common.Config, run func(t *testing.T, input *s3Input, out *stubOutleter)) { - // Simulate input.Context from Filebeat input runner. - inputCtx := newInputContext() - defer close(inputCtx.Done) - - // Stub outlet for receiving events generated by the input. - eventOutlet := newStubOutlet() - defer eventOutlet.Close() - - connector := channel.ConnectorFunc(func(_ *common.Config, _ beat.ClientConfig) (channel.Outleter, error) { - return eventOutlet, nil - }) +func newV2Context() (v2.Context, func()) { + ctx, cancel := context.WithCancel(context.Background()) + return v2.Context{ + Logger: logp.NewLogger("s3_test"), + ID: "test_id", + Cancelation: ctx, + }, cancel +} - in, err := NewInput(cfg, connector, inputCtx) +func setupInput(t *testing.T, cfg *common.Config) (*s3Collector, chan beat.Event) { + inp, err := Plugin().Manager.Create(cfg) if err != nil { t.Fatal(err) } - s3Input := in.(*s3Input) - defer s3Input.Stop() - run(t, s3Input, eventOutlet) -} + ctx, cancel := newV2Context() + t.Cleanup(cancel) -func newInputContext() input.Context { - return input.Context{ - Done: make(chan struct{}), + client := pubtest.NewChanClient(0) + pipeline := pubtest.ConstClient(client) + collector, err := inp.(*s3Input).createCollector(ctx, pipeline) + if err != nil { + t.Fatal(err) } + return collector, client.Channel } -type stubOutleter struct { - sync.Mutex - cond *sync.Cond - done bool - Events []beat.Event -} - -func newStubOutlet() *stubOutleter { - o := &stubOutleter{} - o.cond = sync.NewCond(o) - return o -} - -func (o *stubOutleter) waitForEvents(numEvents int) ([]beat.Event, bool) { - o.Lock() - defer o.Unlock() - - for len(o.Events) < numEvents && !o.done { - o.cond.Wait() +func setupCollector(t *testing.T, cfg *common.Config, mock bool) (*s3Collector, chan beat.Event) { + collector, receiver := setupInput(t, cfg) + if mock { + svcS3 := &MockS3Client{} + svcSQS := &MockSQSClient{} + collector.sqs = svcSQS + collector.s3 = svcS3 + return collector, receiver } - size := numEvents - if size >= len(o.Events) { - size = len(o.Events) + config := getConfigForTest(t) + awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) + if err != nil { + t.Fatal("failed GetAWSCredentials with AWS Config: ", err) } - out := make([]beat.Event, size) - copy(out, o.Events) - return out, len(out) == numEvents + s3BucketRegion := os.Getenv("S3_BUCKET_REGION") + if s3BucketRegion == "" { + t.Log("S3_BUCKET_REGION is not set, default to us-west-1") + s3BucketRegion = "us-west-1" + } + awsConfig.Region = s3BucketRegion + awsConfig = awsConfig.Copy() + collector.sqs = sqs.New(awsConfig) + collector.s3 = s3.New(awsConfig) + return collector, receiver } -func (o *stubOutleter) Close() error { - o.Lock() - defer o.Unlock() - o.done = true - return nil -} - -func (o *stubOutleter) Done() <-chan struct{} { return nil } - -func (o *stubOutleter) OnEvent(event beat.Event) bool { - o.Lock() - defer o.Unlock() - o.Events = append(o.Events, event) - o.cond.Broadcast() - return !o.done +func runTest(t *testing.T, cfg *common.Config, mock bool, run func(t *testing.T, collector *s3Collector, receiver chan beat.Event)) { + collector, receiver := setupCollector(t, cfg, mock) + run(t, collector, receiver) } func TestS3Input(t *testing.T) { - inputConfig := defaultTestConfig() - config := getConfigForTest(t) - - runTest(t, inputConfig, func(t *testing.T, input *s3Input, out *stubOutleter) { - awsConfig, err := awscommon.GetAWSCredentials(config.AwsConfig) - if err != nil { - - } - s3BucketRegion := os.Getenv("S3_BUCKET_REGION") - if s3BucketRegion == "" { - t.Log("S3_BUCKET_REGION is not set, default to us-west-1") - s3BucketRegion = "us-west-1" - } - awsConfig.Region = s3BucketRegion - input.awsConfig = awsConfig.Copy() - svcSQS := sqs.New(awsConfig) - - // remove old messages from SQS - err = input.deleteAllMessages(t, awsConfig, config.QueueURL, int64(config.VisibilityTimeout.Seconds()), svcSQS) - if err != nil { - t.Fatalf("failed to delete message: %v", err.Error()) - } - + runTest(t, defaultTestConfig(), false, func(t *testing.T, collector *s3Collector, receiver chan beat.Event) { // upload a sample log file for testing s3BucketNameEnv := os.Getenv("S3_BUCKET_NAME") if s3BucketNameEnv == "" { t.Fatal("failed to get S3_BUCKET_NAME") } - svcS3 := s3.New(awsConfig) - uploadSampleLogFile(t, s3BucketNameEnv, svcS3) - time.Sleep(30 * time.Second) - wg := sync.WaitGroup{} wg.Add(1) go func() { defer wg.Done() - input.run(svcSQS, svcS3, 300) + collector.run() }() - events, ok := out.waitForEvents(2) - if !ok { - t.Fatalf("Expected 2 events, but got %d.", len(events)) - } - input.Stop() - - // check events - for i, event := range events { - bucketName, err := event.GetValue("aws.s3.bucket.name") - assert.NoError(t, err) - assert.Equal(t, s3BucketNameEnv, bucketName) - - objectKey, err := event.GetValue("aws.s3.object.key") - assert.NoError(t, err) - assert.Equal(t, fileName, objectKey) - - message, err := event.GetValue("message") - assert.NoError(t, err) - switch i { - case 0: - assert.Equal(t, "logline1\n", message) - case 1: - assert.Equal(t, "logline2\n", message) - } - } + event := <-receiver + bucketName, err := event.GetValue("aws.s3.bucket.name") + assert.NoError(t, err) + assert.Equal(t, s3BucketNameEnv, bucketName) - // delete messages from the queue - err = input.deleteAllMessages(t, awsConfig, config.QueueURL, int64(config.VisibilityTimeout.Seconds()), svcSQS) - if err != nil { - t.Fatalf("failed to delete message: %v", err.Error()) - } + objectKey, err := event.GetValue("aws.s3.object.key") + assert.NoError(t, err) + assert.Equal(t, fileName, objectKey) + + message, err := event.GetValue("message") + assert.NoError(t, err) + assert.Equal(t, "logline1\n", message) }) } @@ -354,41 +224,23 @@ func TestMockS3Input(t *testing.T) { "queue_url": "https://sqs.ap-southeast-1.amazonaws.com/123456/test", }) - runTest(t, cfg, func(t *testing.T, input *s3Input, out *stubOutleter) { - svcS3 := &MockS3Client{} - svcSQS := &MockSQSClient{} + runTest(t, cfg, true, func(t *testing.T, collector *s3Collector, receiver chan beat.Event) { + defer collector.cancellation.Done() + defer collector.publisher.Close() - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - input.run(svcSQS, svcS3, 300) - }() + output, err := collector.receiveMessage(collector.sqs, collector.visibilityTimeout) + assert.NoError(t, err) - events, ok := out.waitForEvents(2) - if !ok { - t.Fatalf("Expected 2 events, but got %d.", len(events)) - } - input.Wait() - - // check events - for i, event := range events { - bucketName, err := event.GetValue("aws.s3.bucket.name") - assert.NoError(t, err) - assert.Equal(t, "test-s3-ks-2", bucketName) - - objectKey, err := event.GetValue("aws.s3.object.key") - assert.NoError(t, err) - assert.Equal(t, "server-access-logging2019-06-21-16-16-54", objectKey) - - message, err := event.GetValue("message") - assert.NoError(t, err) - switch i { - case 0: - assert.Equal(t, s3LogString1, message) - case 1: - assert.Equal(t, s3LogString2, message) - } - } + var grp unison.MultiErrGroup + errC := make(chan error) + defer close(errC) + grp.Go(func() (err error) { + return collector.processMessage(collector.s3, output.Messages[0], errC) + }) + + event := <-receiver + bucketName, err := event.GetValue("aws.s3.bucket.name") + assert.NoError(t, err) + assert.Equal(t, "test-s3-ks-2", bucketName) }) } diff --git a/x-pack/filebeat/module/activemq/audit/config/audit.yml b/x-pack/filebeat/module/activemq/audit/config/audit.yml index 3e0ec415541..f40bf16116f 100644 --- a/x-pack/filebeat/module/activemq/audit/config/audit.yml +++ b/x-pack/filebeat/module/activemq/audit/config/audit.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/activemq/log/config/log.yml b/x-pack/filebeat/module/activemq/log/config/log.yml index 17f6bd869f2..55449d359f1 100644 --- a/x-pack/filebeat/module/activemq/log/config/log.yml +++ b/x-pack/filebeat/module/activemq/log/config/log.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml index 5a56f210c79..c4436629e21 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/file.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml index 2094f77c712..ac1caacf21c 100644 --- a/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudtrail/config/s3.yml @@ -51,6 +51,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} @@ -58,4 +62,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml index 5a56f210c79..c4436629e21 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml index 073eca58ab2..bdb0ff350f0 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/config/s3.yml @@ -37,6 +37,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} @@ -44,4 +48,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/ec2/config/file.yml b/x-pack/filebeat/module/aws/ec2/config/file.yml index 5a56f210c79..c4436629e21 100644 --- a/x-pack/filebeat/module/aws/ec2/config/file.yml +++ b/x-pack/filebeat/module/aws/ec2/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/ec2/config/s3.yml b/x-pack/filebeat/module/aws/ec2/config/s3.yml index 073eca58ab2..bdb0ff350f0 100644 --- a/x-pack/filebeat/module/aws/ec2/config/s3.yml +++ b/x-pack/filebeat/module/aws/ec2/config/s3.yml @@ -37,6 +37,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} @@ -44,4 +48,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/elb/_meta/fields.yml b/x-pack/filebeat/module/aws/elb/_meta/fields.yml index 9499f8bbb0e..aac074e9347 100644 --- a/x-pack/filebeat/module/aws/elb/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/elb/_meta/fields.yml @@ -101,3 +101,19 @@ type: keyword description: > The error reason if the executed action failed. + - name: target_port + type: keyword + description: > + List of IP addresses and ports for the targets that processed this request. + - name: target_status_code + type: keyword + description: > + List of status codes from the responses of the targets. + - name: classification + type: keyword + description: > + The classification for desync mitigation. + - name: classification_reason + type: keyword + description: > + The classification reason code. diff --git a/x-pack/filebeat/module/aws/elb/config/file.yml b/x-pack/filebeat/module/aws/elb/config/file.yml index 498a7906457..e9470671e07 100644 --- a/x-pack/filebeat/module/aws/elb/config/file.yml +++ b/x-pack/filebeat/module/aws/elb/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/elb/config/s3.yml b/x-pack/filebeat/module/aws/elb/config/s3.yml index 073eca58ab2..bdb0ff350f0 100644 --- a/x-pack/filebeat/module/aws/elb/config/s3.yml +++ b/x-pack/filebeat/module/aws/elb/config/s3.yml @@ -37,6 +37,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} @@ -44,4 +48,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml index de772ccdf01..8cb2a914921 100644 --- a/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/elb/ingest/pipeline.yml @@ -31,7 +31,7 @@ processors: %{TIMESTAMP_ISO8601:event.start} \"(?:-|%{DATA:_tmp.actions_executed})\" \"(?:-|%{DATA:aws.elb.redirect_url})\" - \"(?:-|%{DATA:aws.elb.error.reason})\" + \"(?:-|%{DATA:aws.elb.error.reason})\"( \"(?:-|%{DATA:_tmp.target_port})\")?( \"(?:-|%{DATA:_tmp.target_status_code})\")?( \"(?:-|%{DATA:aws.elb.classification})\")?( \"(?:-|%{DATA:aws.elb.classification_reason})\")? # TCP from Network Load Balancers (v2 Load Balancers) - >- @@ -141,6 +141,18 @@ processors: separator: ',' ignore_missing: true + - split: + field: '_tmp.target_port' + target_field: 'aws.elb.target_port' + separator: ' ' + ignore_missing: true + + - split: + field: '_tmp.target_status_code' + target_field: 'aws.elb.target_status_code' + separator: ' ' + ignore_missing: true + - date: field: '_tmp.timestamp' formats: diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log index 88ea2d75c26..5d754c4bbaa 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log @@ -8,4 +8,4 @@ http 2019-10-11T15:03:49.331902Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.2 http 2019-10-11T15:55:09.308183Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37838 10.0.0.192:80 0.001 0.000 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5dd-4d9a423a0e9a782fe2f390af" "-" "-" 0 2019-10-11T15:55:09.307000Z "forward" "-" "-" http 2019-10-11T15:55:11.354283Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37850 10.0.1.107:80 0.001 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7d64cabe9955b4df9acc800a" "-" "-" 0 2019-10-11T15:55:11.352000Z "forward" "-" "-" http 2019-10-11T15:55:11.987940Z app/filebeat-aws-elb-test/c86a326e7dc14222 77.227.156.41:37856 10.0.0.192:80 0.000 0.001 0.000 200 200 125 859 "GET http://filebeat-aws-elb-test-12030537.eu-central-1.elb.amazonaws.com:80/ HTTP/1.1" "curl/7.58.0" - - arn:aws:elasticloadbalancing:eu-central-1:627959692251:targetgroup/test-lb-instances/8f04c4fe71f5f794 "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4" "-" "-" 0 2019-10-11T15:55:11.987000Z "forward" "-" "-" - +http 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.000 0.001 0.000 200 200 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337262-36d228ad5d99923122bbe354" "-" "-" 0 2018-07-02T22:22:48.364000Z "forward,redirect" "-" "-" "10.0.0.1:80" "200" "-" "-" diff --git a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json index 28e1564e928..3682fb6520e 100644 --- a/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/application-lb-http.log-expected.json @@ -500,5 +500,55 @@ ], "tracing.trace.id": "Root=1-5da0a5df-7c958e828ff43b63d0e0fac4", "user_agent.original": "curl/7.58.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward", + "redirect" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.1", + "aws.elb.backend.port": "80", + "aws.elb.backend_processing_time.sec": 0.001, + "aws.elb.matched_rule_priority": "0", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.1:80" + ], + "aws.elb.target_status_code": [ + "200" + ], + "aws.elb.trace_id": "Root=1-58337262-36d228ad5d99923122bbe354", + "aws.elb.type": "http", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 34, + "http.request.method": "GET", + "http.request.referrer": "http://www.example.com:80/", + "http.response.body.bytes": 366, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4431, + "service.type": "aws", + "source.ip": "192.168.131.39", + "source.port": "2817", + "tags": [ + "forwarded" + ], + "tracing.trace.id": "Root=1-58337262-36d228ad5d99923122bbe354", + "user_agent.original": "curl/7.46.0" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log index 9e4526d2d61..94c0ec1360b 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log @@ -7,4 +7,7 @@ http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.13 http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - 0.000 0.001 0.000 502 - 34 366 "GET http://www.example.com:80/ HTTP/1.1" "curl/7.46.0" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 0 2018-11-30T22:22:48.364000Z "forward" "-" "LambdaInvalidResponse" http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- http://www.example.com:80- -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-" http 2018-11-30T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 - -1 -1 -1 400 - 0 0 "- - -" "-" - - - "-" "-" "-" 0 2018-11-30T22:22:48.364000Z "-" "-" "-" - +h2 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.1.252:48160 10.0.0.66:9000 0.000 0.002 0.000 200 200 5 257 "GET https://10.0.2.105:773/ HTTP/2.0" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337327-72bd00b0343d75b906739c42" "-" "-" 1 2018-07-02T22:22:48.364000Z "redirect" "https://example.com:80/" "-" "10.0.0.66:9000" "200" "-" "-" +https 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" "www.example.com" "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012" 1 2018-07-02T22:22:48.364000Z "authenticate,forward" "-" "-" "10.0.0.1:80" "200" "-" "-" +ws 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:40914 10.0.1.192:8010 0.001 0.003 0.000 101 101 218 587 "GET http://10.0.0.30:80/ HTTP/1.1" "-" - - arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.1.192:8010" "101" "-" "-" +wss 2018-07-02T22:23:00.186641Z app/my-loadbalancer/50dc6c495c0c9188 10.0.0.140:44244 10.0.0.171:8010 0.000 0.001 0.000 101 101 218 786 "GET https://10.0.0.30:443/ HTTP/1.1" "-" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337364-23a8c76965a2ef7629b185e3" "-" "-" 1 2018-07-02T22:22:48.364000Z "forward" "-" "-" "10.0.0.171:8010" "101" "-" "-" diff --git a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json index eb1fad5f705..2c1490142fa 100644 --- a/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json +++ b/x-pack/filebeat/module/aws/elb/test/example-alb-http.log-expected.json @@ -368,5 +368,220 @@ ], "tracing.trace.id": "-", "user_agent.original": "-" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "redirect" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.66", + "aws.elb.backend.port": "9000", + "aws.elb.backend_processing_time.sec": 0.002, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.redirect_url": "https://example.com:80/", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.66:9000" + ], + "aws.elb.target_status_code": [ + "200" + ], + "aws.elb.trace_id": "Root=1-58337327-72bd00b0343d75b906739c42", + "aws.elb.type": "h2", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 5, + "http.request.method": "GET", + "http.request.referrer": "https://10.0.2.105:773/", + "http.response.body.bytes": 257, + "http.response.status_code": 200, + "http.version": "2.0", + "input.type": "log", + "log.offset": 3284, + "service.type": "aws", + "source.ip": "10.0.1.252", + "source.port": "48160", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337327-72bd00b0343d75b906739c42", + "user_agent.original": "curl/7.46.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "authenticate", + "forward" + ], + "aws.elb.backend.http.response.status_code": 200, + "aws.elb.backend.ip": "10.0.0.1", + "aws.elb.backend.port": "80", + "aws.elb.backend_processing_time.sec": 0.048, + "aws.elb.chosen_cert.arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.086, + "aws.elb.response_processing_time.sec": 0.037, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.1:80" + ], + "aws.elb.target_status_code": [ + "200" + ], + "aws.elb.trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259", + "aws.elb.type": "https", + "cloud.provider": "aws", + "destination.domain": "www.example.com", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 0, + "http.request.method": "GET", + "http.request.referrer": "https://www.example.com:443/", + "http.response.body.bytes": 57, + "http.response.status_code": 200, + "http.version": "1.1", + "input.type": "log", + "log.offset": 3750, + "service.type": "aws", + "source.ip": "192.168.131.39", + "source.port": "2817", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337281-1d84f3d73c47ec4e58577259", + "user_agent.original": "curl/7.46.0" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward" + ], + "aws.elb.backend.http.response.status_code": 101, + "aws.elb.backend.ip": "10.0.1.192", + "aws.elb.backend.port": "8010", + "aws.elb.backend_processing_time.sec": 0.003, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.001, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.1.192:8010" + ], + "aws.elb.target_status_code": [ + "101" + ], + "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "aws.elb.type": "ws", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 218, + "http.request.method": "GET", + "http.request.referrer": "http://10.0.0.30:80/", + "http.response.body.bytes": 587, + "http.response.status_code": 101, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4306, + "service.type": "aws", + "source.ip": "10.0.0.140", + "source.port": "40914", + "tags": [ + "forwarded" + ], + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "user_agent.original": "-" + }, + { + "@timestamp": "2018-07-02T22:23:00.186Z", + "aws.elb.action_executed": [ + "forward" + ], + "aws.elb.backend.http.response.status_code": 101, + "aws.elb.backend.ip": "10.0.0.171", + "aws.elb.backend.port": "8010", + "aws.elb.backend_processing_time.sec": 0.001, + "aws.elb.matched_rule_priority": "1", + "aws.elb.name": "app/my-loadbalancer/50dc6c495c0c9188", + "aws.elb.protocol": "http", + "aws.elb.request_processing_time.sec": 0.0, + "aws.elb.response_processing_time.sec": 0.0, + "aws.elb.ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "aws.elb.ssl_protocol": "TLSv1.2", + "aws.elb.target_group.arn": "arn:aws:elasticloadbalancing:us-west-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", + "aws.elb.target_port": [ + "10.0.0.171:8010" + ], + "aws.elb.target_status_code": [ + "101" + ], + "aws.elb.trace_id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "aws.elb.type": "wss", + "cloud.provider": "aws", + "event.category": "web", + "event.dataset": "aws.elb", + "event.end": "2018-07-02T22:23:00.186Z", + "event.kind": "event", + "event.module": "aws", + "event.outcome": "success", + "event.start": "2018-07-02T22:22:48.364000Z", + "fileset.name": "elb", + "http.request.body.bytes": 218, + "http.request.method": "GET", + "http.request.referrer": "https://10.0.0.30:443/", + "http.response.body.bytes": 786, + "http.response.status_code": 101, + "http.version": "1.1", + "input.type": "log", + "log.offset": 4708, + "service.type": "aws", + "source.ip": "10.0.0.140", + "source.port": "44244", + "tags": [ + "forwarded" + ], + "tls.cipher": "ECDHE-RSA-AES128-GCM-SHA256", + "tls.version": "1.2", + "tls.version_protocol": "tls", + "tracing.trace.id": "Root=1-58337364-23a8c76965a2ef7629b185e3", + "user_agent.original": "-" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 352932f1b1c..e8968b65e8e 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzcXN+T4raTf9+/oisvma0Crr6b1NXVXOWq2NnJhctkMzewyd2TI+wGdCMkR5JhyV9/1ZL8AywDM5jd1JeHXQbb0qdb/VstD+EZd7fAtuYNgOVW4C2Mf5++AdAokBm8hTla9gYgQ5Nqnluu5C38xxsAgF9UVgiEhdKwYjITXC5BqKWBhVZrGmb0BmDBUWTm1j0wBMnWWE5HH7vL8RaWWhV5+CUyD31+dMNUI7t5RuFqc4rmNKlQRWY146K6FJuRPofUlp8MF6wQNnFT3MKCCYN7l6Ngm4CVdnjvCMuMsOxBj8FvkoAblDbZoDZcyb07SkqecbdVOju4dgQYfWYrbCIK44NagF0hAfQTE/o1s6MotMKgTniG0nK7i0I7ZHIb2DCKjEaehIEBBa4JSqqkZVwayNAyLgywuSqsw0uzgVq0xpqMf4ESINgVs7BmGbpHNP5ZoLEDYDKD7YqnK0g1unuZMLBFja3hCoPZCCYLsLjOlWZ613rG3TNwM5S4zUptDazUln5tjdkaQM2JSsxGB7fGhKS5GsSD1sXjMtJejsgNfkUChx1hHUve0G59KKkvR9IWjBLKeM3+UhKe0KhCpwgf2RrhZvz08W0JMNdcpjxn4mDNUybEIVsbqNMUjUmecZfwGL6+8Pt5aCCYfPAIt8w4wQGrwPClbEpoN2CDhpQ2IcXAz7YTckwLzwU8WTSxOKCOnVtuVw01MJgWOiYSsC/ipG6VYjjSc602PEMDXHpbQ2ao1uxAY3TcinWpRmYxc6bWrpTB5pSRR7tUqcnc9YIlrLArGiWl0aN3n5aKcxkNQTo2TBQI3IDV9H9gv1LWGUVQ2hk1931LpHYOFrVMgUX1gjJhlOPhHq1+eVmc7fT55ccxZLjhKf47KLtCveUGB947tgW2yVe3ViS1GbNd4D1Pj9zwEobSMM7IW75G2K7Qa1dbdtsc48YUbUO8T0+phO5efZSgLj18CUV96CO8Xic7xwvu7Xx3Vn6O6SKcdm/l5xw1hBfwGYL0BB8THMtxoRmAKdLV0SGZgSel7ICU+JNBPSCFflKiQ2maDKicWtw7XZsRXFrUkgnyWYEbzbiq6cGW2C0nsC97p8mOxxLXpnb89LGkMkjADUtTVUi/dM7+urXTSuDbo8PF2HNCkM7gigfzdUQhTO4pU1tpricNJb1cbtQzZsk8ZtH6CsxoqnLVKWMzqMnDdSUOpOzAYvEFlDHq/d07GBdWwTRlLjkOueC9YMbyFN4jk8Yy8RxPsFBrpZNUZYeW7/zEL55fNalzk1SBRvArGm2hpXGega4fw7dGY9iyT4iT42B8etUYpDJI3VDDWEnONFujRX24bpeytB54QMxkcjcIukBe0JBv9T66O7JfF8Ly5GSedyzU77h4oojRZJPJlTSYhHigby6V41fxBsWjLKVnTKlnzwjpisklGrjxkf2gnYnnFNY5C5yhQIrw/CBv/4ZMZVnGCRwTiSuqZGyvwnQpX8fV8BTrskby5Es4lUWWypKY2qAurYFKfSO3Ftbpb8jNUpNbru9S2fSZl+OWr3AsOJo9ffURYbCbc+Ry2S7nMCEwgyVK1My657nxQ3fYUFffi0S3FxnQffxl8aYhDyXArCEoGlOlszhMlvOLi5AncY4fJ1UlkhmjUl4no+761oxzfseEaI3kKJgRnUd4vWaSLZ3d8YrYpxLCe6UEMtkhRtsVUprc4DY3cGgFoIHQ39XlzFiWKCniRdeLl6LGyg2onOSEVoQAu6mHNHV9oQujD5zjHuQ1heExCG6c8arGDrU0zIDLmrUvrZxer1xZ1SjHTx/bgeJZwXwfMMYhYK9Tt5KDFL5Hqha91pRPsMYZp0pTdLmSvvDg9qJub9nWDIPdHTpkt+TphvSo+7tDAlOec1L2TgZfojBPmGukuM7bLlbz2Om+xhT5xtlXbo4pc6DLW6Qk7G1cz8ZWYT9NNwAuU1FklJpsCbXVfLlE7d1C3Mj6WpqXoUL8HYNYs2Ias8DQXtf8Pz9NPjRc53zX3EOzCgrJ/yxQ7Ep5bl6PczNsaLqVofSTMjMfygYXYnzuYBVkfLFATX/4/dn9T5A/ExeyTZ4mKLNc8b5ZciBevz3eQTkRqbLfWQsBVCgJulTakd12gPS8VcCkqyo3E9Uq4S6T6+l3cVpTJY0SmAi15PFg5TXeJ+zmmhxTvuApgbzzEz3QPGE1X+p5TmcGx1G3kccdQJ0i3Dsd/kCJAuVep2k4RkeTlrWac4EdQeI+JXMfKnXcc3YpvB2x7FFDKY+TMo8MjiEriXAyk1h1FH4/mz2fnh5aK3CcwQuWFObERtTVWOvsOixYailjrzeLyFJtO6pgEPbMISs0uZhOUksSF4JZi7JF4+vV9n5aD+rSYxduEN/V/P8wtY5A7St7pph7QQemUX5r4VmqrSQrxrINk2krG+5Vvbtob1N5SsUvqwLAeZWAF5TWXkNjd5W2h0Lb+fWufqH3Vf2CF1XAXhZq9kdxj4EnnBV8lmRmfIkmntxf4PmdK7mrGsrgg5sFHtTypV5fqGWy4KKVGtcwJZp4x8FZ2V6VJz+opZunbJ2q82TPoiOSYpm2ieXr7vSvY3v+XF1wM9Dyf5rd+a15TVLv1aCGCEQApIqcd1wRLHumYZivTrgIOfX23I1axaBCLcFzY8U2cRGbI0rSJr5xkrgX3nfzCmV2VU6hzP4p+GS+S+ZF+hzdDrzKFl6ZJoCflkJ2T6JrVCi0bpcAw2wNlq6YOaD3KIU+qrgihTVVfirXR3YTcsVBlPDoWEL54K1qmgsMaRLfTanELbn8ULy+ovBXUh9QrpXxtZWyQgpsrUishTgW0IRcO1i/WsTPNIdKZF+HXD9xB6Xd+Rq8mtJc44arwiRfQluPa2gJZU8du1KNM7SzIm3FzCphYqk0t6v1F7JGNClUk7bbJNz1LvGNseIIocVc8NQ1si64XKLONY9aur7oXOFnlmHK10wAylRlmEFj5qot1+FyBovIjw63ZjZdhYgx13zDLLoHDrpj+ZmsoNuZLTR+kfWul7fVzNuJtu5wMXy5Ol4A7orIT+Gbug70eINtITPUYkehQYjBDa0XkzDxiNr6FiL4shznjZOv5g98Zypb+jsss9xYnpqB29sjStvBiW/Aj2yP2TJMpCQHZVbK0XFgNXv3ToFsSbDeHHL065wC+Z2wvOgUyLEem0i5/ES9YB9EOXibcZi++/ocu797588UcdkAfi7jeJ6wLNNoXr+l0mJf3Q6JFsLo9caaqzigjnBTzM/n5vI6vHx4/yKxa+2tXcy3pksUimUwZ4LJFDvati7qi4gCaB5k2QPgmLR5Bw/04/vwY8d2imV6iTZxqzdqbx9fCLHRfOon8mJSH7HrLG1VFQZuyE8cbuxeiMtJTxi5tclJyaeUmHZ3BORaWZWqw42wC0GVo8bX9GZlbU7uw6b52xOdgVqlaAyXSxfhjwymHU5YtWKXc+ROWSaqVNtgqmRmwPAyCa+556uuvgDLTc3jQlougO/tCVIqv6QlcXk4S59RdjTwhIt/IzIbZNCVABAsF2LvBxcCmFA8zbhcdjac+FLuV6awKvI2167a5N2jcn8tHXsE72wUqFnn6VrHXZpQrba0l1FV1gMaKyVhzYXggdhBoNbDV7nbW2kQlAplDhOxynIKyr9kZlbsGa9LR3nUafYwhWpKYnSq1rmrmB/QBSoipVX9B41lc8HNqou0Uv34YYH5Qgs3eTwMMkohqiXdp0SnLHCJMFf69b1vcSustC1LDJeiI5M9qradKIcozGVN8FHIfmCggdvqedO82lZQ/wD5Q9Jfq7SvrP/RCf2PKMXGiCTl+apvRz2dPoAf1yeiXJIS/Iv7uVqEjtCGMF3HU9P0lbd+Ma50pQzKJEVtrxpx+XmA5uELdxoSQp+Xz+gbQvBa+AY1Zz0z148JsljPUV+ZFi5TtXbuVZiECezbmFButUQdOmnVwllwN0/Do853sRC+1GNPldsgDhR3uyP6liWxncEL6SDYbnAfv3cg0CzFS1qyolO7I+HSVh7jj/8Zjtd/yeGMZhtOsj9ghSzryrp8ES5LdCEonOIqcpT04rDdj1ovsoudClEdt3IQ9k9c8QUFvHRPuNzRq+6DJPyMadHeRr0QeGjOKwf3B4r3XF19cOlmofSW6WwAC/4Zs2HpGQZ7p61Ho9HbEUwspEyWO7VgcIOaCc+eDj3UmHGNqU0K3bM1+fT0ECy043iYxxUO07Ljp2LBkbNhI43M9P3aEn9gzY9cnlurliPgWzAumsjqHTrfMf61CzHT71w/BOrybRAvqcr4zZIk1rp9qdlgUkmeMuHLuHW/uJvr4ESsh9ERy8W2jnosHZX7ReUmePM8fFBGEoYl47KrRqJxrSwmHdF66+dzTEOeM+2d6dnFQWhXIr7YolbzuS4iBkMnnoXcfxVE2bt7vHrSpwMbg7GuX3Cv07reLIw2WiOr+4zjUKsDK/2ytz4g42pjGaz8qyMgw1QwygyYgemv48dRdecAnu6ns9FPs9ljska7UtmoPJDhToIN4Pf799PJ7P7YLUrD+/Hs7qfRh/uH+9n96Nf3/3V/N4uT/ow9u+9vnnH3TbONsHbS5DvCJqAD+c3wm9JK16zKFPpOREs5OXNbfFVj33FJKzTvl5YnP/Dw09NkjyLifWVYWnskTWiU9iU+2+uxlCGLNWqeehzNfLQ+wxPpzOzx7Hg8VarU8N754TuVYXOdpQoOWqWuo6SrWLKzaBLTdQjv1RwL2U9VhHPzuAx+APi5bDl0LK0rxxvUFA43yfgLteowI67tJjH8rzhnL6lb0aCVy/XtPVyCk8DOsol7MtaU0g8n9wpkh66WS1gIvlzZxikWF9Z8ayBHbXKKGjcdEmoLLROmVRHvn7sKfGYbAmxyctaN2H2nCn3ch7jGOd23h97LgZxsPoV5Qpp2OpF17+Jz2+7XhvbJoB6OaaajOWQ4bdF7ZlueD558KOuNlec519mEISbZKZezUj1HNkTA5yFb/zXk2fCdextIJY342aLM6oALJh86KnRVT8tV3gpZDV/yaQBTvvzNoaUv3w/azUvNiHHPSLw6rvTly8QUvNVPd2kZElMibqoogzDwwHao4WY6fXhb1kzroxK4VJZXr5Yj8Z/GSKMLHaWIvYMylx3zj/uNsJ1dvc9t/2SOfxvluLCrn5yu+hMH+/d4LTYD+O8C9W7qQ2+670/6u4zFb3KNQ5INzCjEe/v6pXVa5SftuTBQnjssxTJUMunriTODVpjraNNMM2nc7ogXtGn5uqeb2cP0bWXNGpIWCpuHO4GNk5wLobbnFzCu1Zfz2+MdEJIXlS6uwmNC8iMheVBLU07h3tO6UwUJQ3hfkCM8HCz3ndIl+7mBd9UD/ojnDhikhbFq3fVEhyj1cNg8Hni7k8HVIfNyd7Ncgq5SvUW9uEaFuS4jSLRbpZ/ruRy2uo/XarZY8DTshyudHa/b9gvz4Cx17K0iAd8Axnd3948z90a+++5cWqjlsVzv1UiFWi7J0IZMLzC3XN4B/PrzAD7++mE8GztP/PPkkb53NpNaJq+66uUUjrXftjn7CqkYlKFbNTY3rvLojOJOFR19Rc82MTplWRb3J68p5eWMooOhwA0KuFGaL7lk4m1Z+mxvyQdyuhFmxn4RhBnlitJ79gbM0lwcxbnJ0ytKjDvaT3pYvci7V+thirnE/s1ujd9PcE0SbJonC8FaBwovJGHO7ZqZ55DLVY5DCaG2ZHFmd4/gpr2Fdz9M//fj4B//Rv8Nx3c/D/7xw4+Tj4Pvf3iazuKQr9eg6bl2C5PHzfcD+vdfXYp3/+N49Ob/AwAA//+8xDjD" + return "eJzcXN9z47Zzf7+/Yicv8c1I7nwvmU7HnXRG53MaNc7FtXxJ+8RA5IpCDQEMAEqn/PWdBcAfEkFJtqi7zFcPd7JIAp9d7G8sOIZn3N4A25g3AJZbgTcw+X32BkCjQGbwBuZo2RuADE2qeWG5kjfwH28AAH5RWSkQFkrDkslMcJmDULmBhVYrGub6DcCCo8jMjXtgDJKtsJqOPnZb4A3kWpVF+CUyD31+dMPUI7t5rsPV9hTtaVKhysxqxkV9KTYjffaprT4ZLlgpbOKmuIEFEwZ3LkfBtgEr7fDeEpYnwrIDPQa/TQKuUdpkjdpwJXfuqCh5xu1G6Wzv2gFg9HlaYhtRGB/UAuwSCaCfmNCvmL2OQisN6oRnKC232yi0fSZ3gY2jyGjkaRgYUOCKoKRKWsalgQwt48IAm6vSOrw0G6hFZ6zp5BeoAIJdMgsrlqF7ROOfJRo7AiYz2Cx5uoRUo7uXCQMb1NgZrjSYXcN0ARZXhdJMbzvPuHtGboYKt1mqjYGl2tCvnTE7A6g5UYnZ9d6tMSFprwbxoHPxsIx0lyNyg1+RwGFHWM+St7Rb70vqy5F0BaOCMlmxv5SERzSq1CnCR7ZCuJo8fnxbASw0lykvmNhb85QJsc/WFuo0RWOSZ9wmPIZvKPx+HhoIph88wg0zTnDAKjA8l20J7Qds0JDSJqQY+Nn2Qo5p4amAp4s2FgfUsXPD7bKlBgbTUsdEAnZFnNStVgxHeqHVmmdogEtva8gMNZodaIyOW7Mu1cgsZs7U2qUy2J4y8mifKrWZu1qwhJV2SaOkNHr07uNScSqjIUjHmokSgRuwmv4P7FfKOqMISjuj5r5viNTewaKWKbCoWVAmjHI83KHVLy+Ls50+v/w4gQzXPMV/B2WXqDfc4Mh7x67Atvnq1oqkNmO2D7zn6YEbXsJQGsYZectXCJsleu3qym6XY9yYsmuId+mplNDdqw8S1KeHL6FoCH2E1+tk73jBvZ3uzqrPIV2E4+6t+pyihvACPkOQnuBjgmM5LDQjMGW6PDgkM/ColB2REn8yqEek0I9K9ChNmwG1U4t7p0szgkuLWjJBPitwox1XtT1Yjv1yAruyd5zseCxxaWonjx8rKoMEXLE0VaX0S+fsr1s7rQS+PThcjD1HBOkErngwX0cUwuSeMrWR5nLSUNHL5Vo9Y5bMYxZtqMCMpqpWnTI2g5o8XF/iQMoOLBZfQBWj3t2+g0lpFcxS5pLjkAveCWYsT+E9MmksE8/xBAu1VjpJVbZv+U5P/OL5VZs6N0kdaAS/otGWWhrnGej6IXwrNIblQ0KcHgbj06vWILVB6ocaxkoKptkKLer9dTuXpc3AI2Imk9tR0AXygoZ8q/fR/ZH9qhSWJ0fzvEOhfs/FI0WMNptMoaTBJMQDQ3OpGr+ONygeZSk9Yyo9e0ZIl0zmaODKR/ajbiZeUFjnLHCGAinC84O8/RsylWUZJ3BMJK6okrGdCtO5fJ3Uw1Osy1rJky/h1BZZKktiaoO6dAaq9I3cWlinvyE3K03uuL5zZdNnXo5bvsKx4Gh29NVHhMFuzpHLvFvOYUJgBjlK1My657nxQ/fYUFffi0S3ZxnQXfxV8aYlDxXArCUoGlOlszhMVvCzi5BHcU4epnUlkhmjUt4ko+76xkwKfsuE6IzkKHgiOg/wesUky53d8Yo4pBLCe6UEMtkjRpslUprc4jY3sG8FoIXQ39XnzFiWKCniRdezl6LByg2oguSEVoQAu6nHNHVzoQ+jD5zjHuQ1heEJCG6c8arHDrU0zIDLhrUvrZxerlxZ1ygnjx+7geJJwfwQMCYhYG9St4qDFL5HqhaD1pSPsMYZp1pTdLWSvvDg9qJubtjGjIPdHTtkN+TpxvSo+7tHAlNecFL2XgafozCPWGikuM7bLtbw2Om+xhT52tlXbg4pc6DLW6Qk7G1czsbWYT9NNwIuU1FmlJpsCLXVPM9Re7cQN7K+luZlqBR/xyDWLJnGLDB00DX/z0/TDy3XOd+299CsglLyP0sU20qe29fj3Awbmm5lKP2kzMyHssGFGJ87WAUZXyxQ0x9+f3b3E+TPxIVsXaQJyqxQfGiW7InXbw+3UE1Equx31kIAFUqCLpV2ZHcdID1vFTDpqsrtRLVOuKvkevZdnNZUSaMEJkLlPB6svMb7hN1cU2DKFzwlkLd+onuaJ6zmSz3P8czgMOou8rgDaFKEO6fDHyhRoNzrOA2H6GjTslJzLrAnSNylZO5DpZ57Ti6FdyOWHWoo5XFS5pHBIWQVEU5mEqsOwh9ms+fT431nBQ4zeMGS0hzZiLoYa51dhwVLLWXszWYRWapNTxUMwp45ZKUmF9NLakXiQjBrUXZofL3a3s2aQV167MIN4rua/x+m1hGofWXPlHMv6MA0ym8tPEu1kWTFWLZmMu1kw4Oqdx/tXSqPqfh5VQA4rRLwgtLaa2jsr9IOUGg7vd41LPShql/wogrYy0LN4SgeMPCEk4LPisyM52jiyf0Znt+5ktu6oQw+uFngXuUv9fpC5cmCi05q3MCUaOIdBydle3WefK9yN0/VOtXkyZ5FByTFMm0Ty1f96V/P9vypuuBmoOX/9HTrt+Y1Sb1XgwYiEAGQKnLecUWw7JmGYb464SLk1NtzN2odgwqVg+fGkq3jIjZHlKRNfO0kcSe87+cVyuyinEKZ/VPwyXyXzMv0ObodeJEtvCpNAD8theyeRNeoUGrdLQGG2VosXTKzR+9BCn1UcUEKG6r8VK6P7CrkiqMo4dGxhPLBW900FxjSJr6fUokbcvmheH1B4a+lPqBcKeNrK1WFFNhKkVgLcSigCbl2sH6NiJ9oDpXIvg65fuIeSvvzNXg1pYXGNVelSb6Eth7W0ArKjjr2pRonaGdN2pKZZcJErjS3y9UXskY0KdSTdtsk3PU+8Y2x4gCh5Vzw1DWyLrjMUReaRy3dUHQu8TPLMOUrJgBlqjLMoDVz3ZbrcDmDReRHh1sxmy5DxFhovmYW3QN73bH8RFbQ7cyWGr/IejfL22nm7UXbdLgYni8PF4D7IvJj+GauAz3eYFvKDLXYUmgQYnBD68UkTD2irr6FCL4qx3nj5Kv5I9+ZynJ/h2WWG8tTM3J7e0RpNzjxDfiR7TFbhYmU5KDMKjk6DKxh784pkA0J1pt9jn6dUyC/E5YXnQI51GMTKZcfqRfsgqgG7zIO03dfn2N3t+/8mSIuW8BPZRwvEpZlGs3rt1Q67GvaIdFCGL3ZWHMVB9QRbor56dzML8PL+/cvErvO3trZfGu7RKFYBnMmmEyxp23rrL6IKID2QZYdAI5J63dwTz++Dz/2bKdYpnO0iVu96+728ZkQW82nfiIvJs0Ru97SVl1h4Ib8xP7G7pm4nPSEkTubnJR8Solpf0dAoZVVqdrfCDsTVDVqfE2vltYW5D5sWrw90hmoVYrGcJm7CP/aYNrjhFUndjlF7pRlok61DaZKZgYMr5Lwhnu+6uoLsNw0PC6l5QL4zp4gpfI5LYnLw1n6jLKngSdc/BuR2SKDrgSAYLkQOz+4EMCE4mnGZd7bcOJLuV+ZwrrI2167epN3h8rdtXTsEby3UaBhnadrFXdpQnXa0l5GVVUPaK2UhBUXggdiR4FaD18Vbm+lRVAqlNlPxGrLKSj/kplZsme8LB3VUaen+xnUUxKjU7UqXMV8jy5QESmt6z9oLJsLbpZ9pFXqx/cLzGdauOnDfpBRCVEj6T4lOmaBK4SF0q/vfYtbYaVtVWI4Fx2Z7Ot624lyiNKc1wQfhewHBhq4q55X7atdBfUPkD8k/bVK+8r6H73Q/4hSbIxIUl4sh3bUs9k9+HF9IsolKcG/uJ/rRegJbQjTZTw1TV976xfjSpfKoExS1PaiEZefB2gevnCnISH0efmMviUEr4VvUHM2MHP9mCDL1Rz1hWnhMlUr516FSZjAoY0J5VY56tBJqxbOgrt5Wh51vo2F8JUee6rcBnGguN8d0bcsie0MnkkHwXaD+/i9B4FmKZ7TkhWd2h0Jl7b2GH/8z3iy+kuOn2i28TT7A5bIsr6syxfhskSXgsIpriJHSc8O2/2ozSK72KkU9XErB2H3xBVfUMBL94TLPb3qPkjCz5iW3W3UM4GH5rxqcH+geMfVNQeXrhZKb5jORrDgnzEbV55htHPa+vr6+u01TC2kTFY7tWBwjZoJz54ePdSYcY2pTUo9sDX59HgfLLTjeJjHFQ7TquOnZsGBs2HXGpkZ+rUl/sCaH7k6t1YvR8C3YFz0hqA+eR82CLoPW+5NsIbGVTZpGlPHon5uUx+zDt3rLj46mNEH0BeJhirsrZin1ZZaCa3ZrUn0+TnBjPHOZvD31eyO7XiaodnKFFbc8vzAYYTdJ5NLSOUeuCCexMtuJdB8588tfO1y4Ow715WDunonyUtqg37LLokdIDiXlUwqyVMm/GZCc2rBzbV3LtvD6MkoYhuYAxYwq13LqhWj/VaGRrlZzrjs02uNK2Ux6ckZOz+f4qCKgmkf0p1cooZuPeyLLWo9n+tlYzB24lnK3ReSVB3kh2t4Q4ZREzDWda3u9Ps3W9bRdn9kTbd7HGp9bGpY9jbHtFyFNoOlf4EJZJgKRvkpMzD7dfJwXd85gse72dP1T09PD8kK7VJl19WxIHcecQS/372fTZ/uDt2iNLyfPN3+dP3h7v7u6e761/f/dXf7FCf9GQcOIr95xu037WbWJlSkCCZsRTuQ34y/qWKFhlWZQt8Pa9kzAnMbzXV76WFJKzUflpZHP/D40+N0hyLifW1YOjt1bWhLa4sQIAxYUJPlCjVPPY52VaQ5SRbpDx7wDQbxhL1WwzsXDd6qDNvrLFUIE1Xq+pr6SnZbiyYxfUdBX82xkIPXpWA3j6sjjQA/V42vjqXN/sUaNSVlbTL+Qq16zIhr/koM/yvO2XOqpzRo7XJ9kxmX4CSwN8JyT8Zao4bh5E6Zdt/VcgkLwfOlbZ2lcmHNtwYK1Kag3GXdI6G21DJhWpXxLs6LwGe2JcCmIGfdyiC3qtSHfYhr39RDe+idTNzJ5mOYJxQLjpdT3BshXfPHpaF9MqjHE5rpYCUjnPkZvL5SnVKffqiq3rXnOdXZhCGm2TGXs1QDRzZEwOcxW/015tn4nXsnTS2N+NmizJqAC6YfeurEdWfVRd5NWg9f8WkEM57/5tDSl+9H3Ra6dsS4YyReHVf6InpiSt7p6jy3GI4pETdTlEEYuGdb1HA1m92/rSr3zYEdzJXl9QsOSfxnMdLoQk9BbOe41nkvm4j7jdBUUb9VcPd8mH8n6qS0y5+crvpzL7v3eC02I/jvEvV25kNvuu9P+ruKxa8KjWOSDcwoxHv7+qV1WuUnHbg8VZ1+rcQy1NPp65GTq1aYy2jTk2bSuD06L2iz6qVjV0/3s7e1NWtJWiiv7+9Ht84TL4TanF7AuFR32G8Pt0BIXlS6uAiPCcmPhORe5aaawr0teKtKEobw1ipHeHi9ge/Xr9jPDbyrH/AHjbfAIC2NVau+J3pEaYBXHsQDb3c+vX7VQVXXrJagb8PIol5cYp+jKSNItBuln5u5HLamm9xqtljwNHRlKJ0d3j0YFubeif7Yu20CvhFMbm/vHp7ceyHv+nNpofJDud6rkQqV52RoQ6YXmFst7wh+/XkEH3/9MHmaOE/88/SBvve2NFsmL7rq1RSOtd92OfsKqRhVoVs9Njeu8uiM4laVPd1tzzYxOmVZFvcnrynlFYyig7HANQq4UprnXDLxtip9dhtDAjn9CDNjvwjCjHJF6T17C2a9DXII57pILygx7gUTpIf16+QHtR6mnEsc3uw2+P0ElyTBpkWyEKxzrPVMEubcrph5Drlc7TiUEGpDFufp9gHctDfw7ofZ/34c/ePf6L/x5Pbn0T9++HH6cfT9D4+zpzjky7UJe67dwPRh/f2I/v1Xl+Ld/Ti5fvP/AQAA//9d8O3V" } diff --git a/x-pack/filebeat/module/aws/s3access/config/file.yml b/x-pack/filebeat/module/aws/s3access/config/file.yml index 498a7906457..e9470671e07 100644 --- a/x-pack/filebeat/module/aws/s3access/config/file.yml +++ b/x-pack/filebeat/module/aws/s3access/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/s3access/config/s3.yml b/x-pack/filebeat/module/aws/s3access/config/s3.yml index 073eca58ab2..bdb0ff350f0 100644 --- a/x-pack/filebeat/module/aws/s3access/config/s3.yml +++ b/x-pack/filebeat/module/aws/s3access/config/s3.yml @@ -37,6 +37,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} @@ -44,4 +48,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/aws/vpcflow/config/input.yml b/x-pack/filebeat/module/aws/vpcflow/config/input.yml index c9e88b6a743..628196b7d3e 100644 --- a/x-pack/filebeat/module/aws/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/aws/vpcflow/config/input.yml @@ -39,6 +39,10 @@ session_token: {{ .session_token }} role_arn: {{ .role_arn }} {{ end }} +{{ if .fips_enabled }} +fips_enabled: {{ .fips_enabled }} +{{ end }} + {{ else if eq .input "file" }} type: log @@ -173,4 +177,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index a4567959194..77c7ea3f5d0 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -13,4 +13,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/activitylogs/config/file.yml b/x-pack/filebeat/module/azure/activitylogs/config/file.yml index 498a7906457..e9470671e07 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/file.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/file.yml @@ -11,4 +11,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 3633cc4e5de..43a051f8dd3 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/auditlogs/config/file.yml b/x-pack/filebeat/module/azure/auditlogs/config/file.yml index 937446eb523..8e77b860e99 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/file.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index dd8e1473a68..b971be2783c 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/azure/signinlogs/config/file.yml b/x-pack/filebeat/module/azure/signinlogs/config/file.yml index 937446eb523..8e77b860e99 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/file.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/file.yml @@ -10,4 +10,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cef/log/config/input.yml b/x-pack/filebeat/module/cef/log/config/input.yml index 49a2b1829be..cc911a8611f 100644 --- a/x-pack/filebeat/module/cef/log/config/input.yml +++ b/x-pack/filebeat/module/cef/log/config/input.yml @@ -28,4 +28,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml index 12440f8fffe..f447d2aacdf 100644 --- a/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml +++ b/x-pack/filebeat/module/checkpoint/firewall/config/firewall.yml @@ -23,4 +23,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index 08dc160fab0..aaf285680e0 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -5,13 +5,15 @@ == Cisco module -This is a module for Cisco network device's logs. It includes the following +This is a module for Cisco network device's logs and Cisco Umbrella. It includes the following filesets for receiving logs over syslog or read from a file: - `asa` fileset: supports Cisco ASA firewall logs. - `ftd` fileset: supports Cisco Firepower Threat Defense logs. - `ios` fileset: supports Cisco IOS router and switch logs. - `nexus` fileset: supports Cisco Nexus switch logs. +- `meraki` fileset: supports Cisco Meraki logs. +- `umbrella` fileset: supports Cisco Umbrella logs. Cisco ASA devices also support exporting flow records using NetFlow, which is supported by the {filebeat-ref}/filebeat-module-netflow.html[netflow module] in @@ -27,6 +29,8 @@ The module is by default configured to run via syslog on port 9001 for ASA and port 9002 for IOS. However it can also be configured to read from a file path. See the following example. +Cisco Umbrella publishes its logs in a compressed CSV format to a S3 bucket. + ["source","yaml",subs="attributes"] ----- - module: cisco @@ -374,6 +378,59 @@ will be found under `rsa.raw`. The default is false. :fileset_ex!: +[float] +==== `umbrella` fileset settings + +The Cisco Umbrella fileset primarily focuses on reading CSV files from an S3 bucket using the filebeat S3 input. + +To configure Cisco Umbrella to log to either your own S3 bucket or one that is managed by Cisco please follow the https://docs.umbrella.com/deployment-umbrella/docs/log-management[Cisco Umbrella User Guide.] + +This fileset supports all 4 log types: +- Proxy +- Cloud Firewall +- IP Logs +- DNS logs + +The Cisco Umbrella fileset depends on the original file path structure being followed. This structure is documented https://docs.umbrella.com/deployment-umbrella/docs/log-formats-and-versioning[Umbrella Log Formats and Versioning]: + +/--
/--
---.csv.gz +dnslogs/--/----.csv.gz + +When configuring the fileset, please ensure that the Queue URL is set to the root folder that includes each of these subfolders above. + +Example config: + +[source,yaml] +---- +- module: cisco + umbrella: + enabled: true + var.input: s3 + var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + var.access_key_id: 123456 + var.secret_access_key: PASSWORD +---- + +*`var.input`*:: + +The input from which messages are read. Can be S3 or file. + +*`var.queue_url`*:: + +The URL to the SQS queue if the input type is S3. + +*`var.access_key_id`*:: + +The ID for the access key used to read from the SQS queue. + +*`var.secret_access_key`*:: + +The secret token used for authenticating to the SQS queue. + +:has-dashboards!: + +:fileset_ex!: + [float] === Example dashboard diff --git a/x-pack/filebeat/module/cisco/fields.go b/x-pack/filebeat/module/cisco/fields.go index fac83c30f27..de57daee50f 100644 --- a/x-pack/filebeat/module/cisco/fields.go +++ b/x-pack/filebeat/module/cisco/fields.go @@ -19,5 +19,5 @@ func init() { // AssetCisco returns asset data. // This is the base64 encoded gzipped contents of module/cisco. func AssetCisco() string { - return "eJzsvW1zIzeSIPx9fwXOEc+52yGzx+2XvfHN7oVWkte66W5rW93tfS4mogJEgSRGKKAaQJGif/0FEqgXVqFIiQKK6r3xB4ctkolEIpHI9/wW3dHtz4gwTeQ/IWSY4fRndOH/N6eaKFYaJsXP6F//CSGE3sq84hQtpEIrLHLOxNJ9HQlqNlLdoZyuGaGIy6We/RNCC0Z5rn/+J/i1/edbJHBB/ZozrHHzCUJmW9Kf0VLJquz8NYBG/c8vAB3QcVic356jX5iiG8z5rPPVGo32LzUeBdUaL2nG8h3IDpU7ut1ItfvJHnQQ+rCiHUw8bMRyKgxbMKpanAKo6GqxYPcPRIPe46K0p6Wp1kyKh+P4G/wdc78ewgtDFfr/LMIPRVRWitCMCUPVAhMag3K3ABM1MOFQzYqiBZcbJBWiayrMXrRyqg0T2MKPi9tlC/hJCKqK08z+Zwyk3uGCIrkAFM4JoVqjCymMkhy9YdrAYsissEEFNmRFc2RWTD8AS3+6laYqBa4WrsOLafiDW8+T80EYdg96MjQ7iz4G1wKXJc2z+sqUATx7fzwoYIzCQnNsaF7T7voG4TxXVOtH4LKS2jyYagtccZOBGP0ZLTDX9Kk42+UfgW0pVQhbLsXyqZhY0A/BZEe+RD7ILnc97jS7WJ3qSLvYP/Rcu3inONwuTgdP2KwUxSbjdE15HD3AwkMAD6RFgfkGK4peobk0ghqL6WLByAz9JkDmrKnafsvl5gzZf/XAFTKnCht6hlZsubKPDXzd/s9DtkWwoUuptjF2duFhNc/f+M5+sY9iraasmar0mf9Of39Gyb9jcYaoIXv3Q6QQlLgLGEVf+yjY56qroMG2MLzpezFhpCgzu2gAC73qs/NeHK4v3t7ALw8vSGQea0EL6qG0HtlnGrHy6eZdZ220s3ZIF8BlpiiRKtePQ+QJGj7Wmi0FzdHl+Q3qLx4kZVFgkWecCZphtawKKsx06PrlkV0eNctbE21JczTfwjXmkmCOcJUzYz/Zt516+/1H8IF7eOwr2T6HLeGNRNhxCmdUGKQr0IAXFefbhnvE3l2Uiq0Zp0s6k/yRPHzkWfy+ogJh0Cx1u7zVL8kKi2WtoXt9U/IcrTGv9nJ/uwlBN89wE4JuDm9iXiltZnL+d0r6UizdpVDUqQluWRD7gAfaYCWYWO690A5jNg3bdLG1SgC6vjwKXVIpRYXJLIzpZI9b1CML6GtKxQOwlWLBlpWi+WkQbtfv4H4YbbxengZfvKYKL+mTCH0y5DvEDuwDcy43NH8IhxcVx4ataUZkJaYTJkYazBGsaXX5Du4rZjTSTBDqhLqTNhusEbGquRVAChFOsepscNdHujBdbJ7uI/2FKVrKDVW1lXJJF1Ro+kwcp798uPyyHKcW4X84Tv/hOP2H4/SLdpyij5qiq4tb/9FMYDNj5T/8qUf6U0PkfMaO1gbdzuePYIF/OGEP47TLFn06/8NF+w8X7T9ctDsLHnTRakoqxUyIaYLelHbB3nLv8cZr+kDcWw8XXdlXen8U6gt3E6dEca+beNfGY1JHtfGuf7ttUnDqf8ZNOQxacMbZIx6uB2qD9ol1OraFvpeTFpgwHubmvXbc7dXF486lXggZiTYrRlZOSHqbU9EFVRq9WLSi8Qzdvnt7c4Zu///bM4SFVXR6YBdSmdXLGTpvgRMs0JwijFZY5SB+XWrUGcKoVNJIIvkZAlFWuKwquejLXKvkb7WhBdJyYSyQGbo2KKdCGrpjBHhJT3ClG9q7n/bfKbfN2YARfQLXrLHTZj3rQK6p2ihm7KOlKjrg1+EhHbhCew6qy0J1ZllrQG5WVDl/in/I0AprNKdUIDnXVK1pPtyf2kk1O7SZ4eXbu5XxuwVYC7yrsoyvPrZ+aImONqeXvWPet8K+W9U7lQ/WVrujW2vLVdoFXgguTeXpr/CmuThg8xFZUG03Le3nPdAIvZFLdEntw6bCG3GwWB+pY7dTwwVz02q7JDJgj3Bi6nuSuxtPpDAQwJMLxIQ2WJgaDR3E0bDiGATzvic4hJ238e0SCBsvTnHtW3PuT4zeUfM7M8I+A/70ZwPWaDarV7LiORJ0TZWVoDXflVhpit5Sgy1qGC2ULDpLvXgjl/rVDSZ31OiXA/CXTFFi+PasiU9h9J46YeE4XHTQnAUJObQ9HkbJgQnVo+QlLRUlYDBZTHK6YFZtkIIDWgbPuVXiyzBWhV72Ve24HOjP+K2/59eX37mYnvfy1Iq5+xa9xwQiyO681OAgYHcMlDbHLfA9exwlVoaRimMFv/cHOxvljAHoozglxBkDyOOcMnok62nP5PU/zmT/mdhV0xzI066vnP89g430j+XZYLfGxwi95Kgp6nTf54ibJVuq+/80zLTBhha0Fxx9JshB+lFGOO7d4WeCHhWm56F7JoitBl6nZ4IYE8chllZjqiXH8+W0nOJjpEdasi2oixjEsqFG9JqQndn5Yu0WsNgM9JCBkvA0K6KnhwygH7Aixqk4CLxOQkXR8aoEyefINdhmJPKhAAUfTT4yhVpdDWIO9f79n7a7Ru2FFMQ+DtjI527ZjoibNUsrDrvUvbDLsAUjuHuf38ilizfUGS2VyKkCZyn1gmqw9QW7pznSFLKudn68u4YeN1jqQxjAfrLB0hzCAPSjDmXoCYzvXzqOMQf7egRNHkeDQUg9CV/+KrXpikje50hNRc7Esv5Qh9im40P6cujLjmGwwY9GCXt9s/6hyeEfu+594g52b+SXStz1T6nJ+9P/u+QdRJ2TyIa+XHCOtK63LEcYLdmaisZJ9uUqApZEx/kv0log+XNU/r6MiMaoQ0OW20zRzwnOuhs8hAOGfft6syu3NLqBi3TmvdkGow/bkiKChxJkThFlZkUV+ngtzHc/IanQL1xi8/1rNMcauKgOkEE1Aah+B/Z9jLr7Be8bwqDpjM8I/oVgItwk1nG98hfvYJBqg9WgOjOa1tGRaJ1tdyl5ffNpR9/DUL/WP1JU57a4R9SjDen21HGqdsSDwhnFlgxqL9xvdrWVA3RIpX/tSYy4vvn0U4AE4ZwcFIEEDUZDKsd4fVpGHSqOx74+K4pzqiaJXf8KS6Hry6dESR2+3WApgDkuVvqsnWycZMn9bLhWtK5bRQsuijVdLiTnlBipvkQBbKl3gpwby3NMI+JIR3OL6Y6i+kb21Ra0h9DP0OIryPy5qKqF1JDsVkiB5tvBoSGk6OeKaiiC0qwo+dafk/0yJOpSTFZIs5yiF39CZqUq9PrHH19CaaimVDSr7KHEs1BeH0AJXUqhaTpSkC+GK1yJcO1TqIq5E3r2KusgBPQCz+WadojBRDCzshZv2iiKi9H7Q74YtjkxqWjOqr6eFoNQX4U0x8axwBaImb9Vr//03Z+1E+mvShCgNdJ/G+zmb9YefIO3VKHX6EoQXGoogpcCTMpHyfUQ9CcGPwK5laFVvn+N/sVu9wx9/z36F0SkgpYXcExu0TP037n5n/aLTKNdonwVPEIh80DR8DOxdcWGZgRzPsfkLq0G7JCrCwawcXaFJSIVeSmZMHV3kSCiwBwZVUomyk9r9UFdUsIwB4wBU22kspq12Dqtw36wxpzljjFCSCG0kJXI7QvDKSDPxNIrRweTF3dvxAByjFigvw57wkYjp7DlEufP5Z3z6CDN/qCooEYxErA6vCnc/TLYwu65r4WwffaxaTVauaiPbYZ+lRt7NEObkwkklTXGjER3lJYHiPYsXrwvhGhKQjHYmuVZnirqelVLniUVUDWroayqcna0twvXTJkKc2u07/jeRcDFwQpmzW6IlQMx3C78Vb++RMpKaw0OFSAaVktqmq8dpIRWiZKeTk6JumZ/HyVUklDQUPBfX9a+1/e0kIaiW8/vda+c+XZMUCLoNuICMV9A4MWvlOmSs5SZDc/anNdsoPY/C93MytyE/A63zr4BdZmm57raavFPyH+NCKMTLwvGTxCjt6ta4+jm4vzG676+KJcVpVR9jRfBE/nFpUFUz8P94ds0gCE+bL6GnCt115Sv2p+0BrvTc8Ayn6HXP/6ENkD3gmKBMOdhXwE49UFNav1HaEOV64GHoM0H1gZJ0SsX2SXiydXEL5uIgbuaImzrafe7VDkQDrKaKFkJyeVy2w/ELZgaaLEI/YjICitMjCOivdRbwB+c5gJVwuf08B2f+WhFbeyCbheoTxlE2BO7BIuisEqmFHUYQeHNqEwDydpTKzEBjdXFKIT3OUhCoMcjQNQGixyrHAmpCszZH6H8XqmKIH1yn+VwNIlkNR88SY8iUot1g8wrzhYUdhww8DUlUuQjCnZ73Jk2Kf0sezbEBJFFyakJMsCoExWDAm8U64nBTr2ZMidi5Fu7dpCdx1h5lzNH2a+QwqwiHVNbnxor56XNcspPRPgrkacguwX5hxSpuy3sEYt29VrFdOm1H/oUHoioZDf6HBl6b/zlQ2uqdKecIt+XBxY436cy25biWNtsy/SIVDnN072DPsnGP1O6WbHWMepMm+aL3fj68LVSspgB1AqK8jWhAismnVpfVNywbw2jCuGy5HX1S9vLpsACL0OluQhxCO/stPWpG0ohZr7WSG6Ei4wZXJR9z6DHGPpvKjlMPmJGI7Ji1rqROdUz9LbSBsykLlB7K7EZycvFhh55SHsF2GJh8V7TKTQhOOR6QUc7aAVFBXEMga1qnbM1y61mA/wQFmS3tSD70CNeeJP3JVOT7bA9TxcLurecyAzf1n2vjAR9zSLlmjPu9Y1GPPRRF86ZlcaNPJsNlmzSyWQVWwIVA0XuqRAb+se+KqBBfq5oNRkrWe52XNTKxw3WCJDIR/gGkPsuNlEjKgU7BE0g05aFSfD6LosUuJZZAlTLLIX2XMYURbtAX0eHmkBX6rwipzEhe+Zj8I0ZPJePenOOFZuH5NoxwYL2geh1Q4jtCMJkoMTHUKx1xVOHnUasKFkZIgv6yuHQGC+QlT1ogIksXzgS7BiQIwxC13TQDneyjdWr+yLATmRnn8snbfHioHege6WbShcLDeJOJSVswVrDJ6zd+lbuIzzldeX02UyBA2hcjCxvCyZqF1XugyxBvL3ZPNUhfNq10ruWoFTot1ufGst0nRDQ96sh3xe2N0AB7VRJ6lJqFlFwPIi3wJwWueswBan89d0d7cJTcTNsmH0qUSSqgipGHiuLgnuboIptz8a6lWzNzXBiyd3vwdbWVORS+YTZvTuT87+foHtNHdoNtDXvIpa+FnxAbitB9yPmJH3KXnVfDS+kr/r3YsZ7uVa4yS0W0iAMYyIskuEEWi6XWZ2ochKhXjPio4X6FD1TdmTfv0O6FXSt3h132MWqlJyRberbs0cu3AACvrm24NsRuRwctpSYgO8rTgGxsDiVwtD71Bprg9C1cP66th8qznNt/wWPKox6A4RCDWAOPM5uSmbWH9eZQBaMBS7rkZxNrxBsjGLzytCOhBjm6PsBn1Zb7z5/YdGhy/4EsadbLW7U6/Q3BwzBfn6Rnzvb0d8Cxi1UgFmC1Q0HdZvzpdZUzdAtdYdSaapmeEmhlbfPdF9IVeMwgF2DcXo7cUO33O87fSukQnMlN/az+q9e13Rm12g/6ev8BisT203XAI7tUfF3qj/Hd7o71czqTXilZEl9QDHVW3wuEOZUmSa7SLWL+r+58JYXH50mAJCEFFCYcySk+FbRkoIlsy/7AcyGKZ+cevhoY6+YZkDnK+YibHX4Z7CzDTMrryw7WY8uYcE5VJsIJMW3S2n/e89LAEpKFlAcE+4bd4KBrwABi6RcICsdDKN6hm5bmdIfbNCtrEqD8YUr56u0NWJcyahLtsm9+PWEx4jwSpuaIf3/DI4JfsK0PUlfE+39G1bxhU/HVaDJtR93w8IWvWvLlE4p+/qQ4WWxvAQsENZaEgb+UnsaQXsSDuwNu6M/I4zK1VYzgjnKmb47Q6WCmSgwSuzrsKKMFT6m9vKRD72rs1G4oAaGmWMNXbw0NHJwvQjq0flyJ2g/LK3ZmYqGhk+Tew9OpfF1zjDBw+TEN5FFWQ3vYIJjw2jDRC43Pp+WSEFoac6aTIpRYgy2uag436LPFebO+ZnLAjPhpYboLMTlyNPV9XrGUpf2bN2qhG+YuKO5rwWqE9GxBu+UN1DsJ181qM1Yvu/g+KArRFJR153s5NwSfQRq9H67PRVev5Xe84puh+16mqAzVQXrD3ZK7WL1awK2jv/3a9rfR9a0F4ynv+PNln+B1ZprrGheEYrqyBENu9s0VQzzLPCaJntEbmHJWm3uv4+dB9C+MKN+AUru9FEtB2J4jP3q9qFbYb1qbqhVCwNVhhVZuczfusamKTO8qCH1WoTZjTTLzLQiMPi+/v9hpSmy8lwgBjl3lYAR+fZP0AivRc0XELZD8Fxh5+HogxN+1bDP07N+sYgs5vU8XbnYebB82ah6xOsFA1+n9vR1tRFAYNzjN02ANHAlLtzqrifjuKfUWXDJXeMN+ZyX+foSvXOS5oVv3IDctD1f9GtxexnWq50D+hS+/I77+foSSOpL3hoxMfQe7EbkXBqg28LMMZGVBRumw0bqWm9T9rLfjer6Am2nLuz1Y48MR0586S7aSbnXlwc12Vj+uQOarEXstchbjXaGLlx9pu93yt0H+7VZQFDtfuO7r7w7bl6ZpnJTmuYxqgSn2lFGugdlI9EaK4bnfFAF6JoyMIFKjkcEgaZCJ+2PsnOgXVXVrTyzkspqGHV9IbPnfPvq+qavQyPfMtZ5FMbqso8cKPjgWsg20uKQRNfCoFu2FBiExQiLllKlbF779UB+WSa9qXU3CV0d4T8tIt3h05bLchlgnHe/fUBMEF7l1IozP8jWDcJ/cVUPML5xDhEHFqT3LOwXgcjc5LFNcE61T0sYM6bvrMp9BF6PKMXruDHf+afhPdN3e0KuRrHlkqp0I+zCJPvUjQV4HNyIZkX1SvLcco+z1Ucmje6E3ifwLAxj714qv3jvdIyXTTOO68twGcmDo/NEFmU2cd4VnIrPvYIxrs6/p6v5txYdKaA+deFmc+cVGbPSvFp6oqyxLuaNtJQKOg9YuV7jNzIlzg8iP4kCOOyqv4DZ5+4hspsYaY38wgpRjN5iUvdTDiu3VgRNasdI8W2toKr9UsjZmtGHWiuKdfTcYG2wqWIpzo0/CjN+MrPDLj6X94jlr8bfL/uyVlNgaDH6OGh87O6CxSJ8det3LPH0vQGTXw7n7h3znDEhq1gxzk4diV5Gv1NWksZ0Ogw8sj9EBpy6M+MOS5xzbuUe0hUhVOtFxdGVXR8RmVNtWaJu9hu2LJjI6X1kAnCmzXGa5xNlCywMppiqkZhTBfHNAivGIYMn4MFz8XexRBiI+K39bXBnIgEfyrlrLnQijdivjl40+ZwlVbr0RbdOwgxI5lWENiG+7vD0cqTI0Lm5hu9x6oQSp3w1SV7eV+W+bT/ETGiUU4MZDzgZ5rIynd+NbE3yyXMza48tbvLYAI/xh9TQouTJsnnOUU4X2IeAfOfLOobvszWtVrymiuMtFHIZ6R9X9CJwI+0HYHX7X9NFXQXufPXaMFNBY0YU3FhrGwwbNj31ukaNYnX8OwTHxjSBrCKyKOx9SsNGFw46Yp1k31LJNcud/6zuIldQPZoIlUtyfKDx8d6yXxhvtUbSzcsLqwb3JSQ9nUbW16unlfV/l/Mj/U5Hb+9/y7kPwIRvV8nSNc69hIRid/K3N9foeqBQddFI1rXWV5fsxyBiYVdTDbuMakg/xh/mc6vDyr0TEdlc5qkrvgYVd32lw+OCLC4j6tEqfrcEFzKYoPK84wL2pcMugbaJh7Aly5tQzogTr4htNQ7KwCO8/PGUvGbfZZXymaqne998dN1z6kAUJGvcU1J1vQgu9WtOQ+WtdRemfYkbEzhCgl7xfNch0lRX4jVmHA8DGahxhSOor1xQpUYmLbg7dIyvP17czRsrhW8A5QKwgy35dAPNlrMRiciKbF7l+Ta6f4YVWdQ6oA7cStPjGp3v9VLFh6iYjNjloFdil+lqioIEprvZq67nKq5yZprKurYvmscoNNiurdhwoqQNL+zfpMsSi03B9WRW+cWnK/TC10p8qrjVleeMQwEH5IFd3ZdS22++RN8OHQ2iH4W5E3IjdgwhTUkFzSzWu9BHJm0SPIELrp8WelFXub/zpUlv6BKTLfo4aq5xNlf4FEX5fuEdEjOBCszEQuGC7k3HKLGCqb3p+yTsKJc3sCx6J3OXHN22BexknQWQQge0L0gVsIRIZSHt9o17Rzfo10qAKflW5pSjF0ysZ9+cISbJGZrbf1H7Lyww32qmZ9+E44uGlNmC48Hk/Ng61K6Gf3GDYFHwdYGc3NbDr+Rib6MGI5Ni6v4693jWbRA0VZaRgwiti7hyt4fZp7e/Y0XRB5cA/M03n97+fv7+6ptvXM7tGivMRnlyI9VdzJLlgxfs93rBboRt1AmGRWwlwtfsxO1S0jwHmNjnYpvAhFlIRYVmJKYA6biSEmBcxPeCBOIDsYBmG8yGw4mf7B2A3uexgdrrE7tEXVfzRJfCzHNtVOzKd6jXTuYQ676l0d7RuuYjnZP02GKXdjDYQKXxxSZt3Yuvd7EgFmzU0VRvNZkj9titBrsRBbbZL+8JC+Wj+wk+3nFhkff6//vhqq3K7Cb/nYTF8o6P3iOyF8mTMEcdx92Hn5QTJG3tnGzHLn1hmoz2OssO+mS+BLfbgHMPR6brltVsingYFH0tMOOW1nUzlxsvM64vu7Vt0InLmoOGLgMtDMazCuuc68yqiEfs55jEa0i39tVHF7IoKtH3RA2wE8c1bnoqdu/ovfl3GtapG9z0cZr1U3G7xSL/NxmOmrW4GWzYMZLhydgNF95BTle6ZITJaFmiU1nwgP0GKzEMOjx31LUoykymEsa3797eoN+cH7VNSg0j8nnSVILb/3iDPldUjfRurbjIFO136kyb3NBxiG7R+7roLJjW1WjpJOJD2gUqY48RsEDLoxxHh6CaQHDsyXDz+AMaMMeqSHBaFmwC9wIuIxYgN0CrPNpU2h2Ycbtd7YDOselrhU+FO6eCrAqsYpWVNHC3JR6ML35y9AmTQTpVFJjZKjovELqIW0DVAF4sodVSArBy/vcEUEscfRKG6zgVnb0g6J6x2A+O79xWUKt6RkdaZJjAYJT45ScWthYRjfcO4PmyXP8g7s0q+vtOREaMynIdte96B7qFfFzk6QGA1xxHlxgio2LJRMSiyCHoFLnRIltkesMMiS4/RLbgcqNxET93pQtbmHU66AmiLkRkTKQUJ0yUVBXzbbSE9wHsktylAb7GPAWvsDIrlTQyix+SAujrHzLwOMaHzZPdTS6XWZ6C2BZw/Pw3IrIC32fGxHIb7AK2HM1pgkehYCIR0kykQ7rkOuNznsUOi+7A/lNC4NE7g3dgx+6F2IUdu6q3C/vHhLB/Sgj7nxPC/h8JYf85DWwjS47nNIVIaaDHN89EVlQclO/5NsE7WQMv7xLoJUXF2bIo02jfVsvEfBk7CclDZimUEk0/k/i+EZFpl5CY4AS1ImmsSQs4jTWpt7oqE8wiJaIpq05iqhpprOlB7xOIECONNcxSwQazJgnwSrB7gYXUlCRgwvVPliqJHoX1T7I0K4rzBG41WZQZ4Ql82BZwgiAJwFXzrYnvFrWQdRLIZZUliGkQxQwjmCcoINIZXlJBthGzrrqwBebbP2g+T4H3OoM2oEkgu3YwabB2ibVJoM+X5fqnND5onc2Z+XOSRmNEZ3FnxfUAKxldVOsk1xygUqLiV7lp5+OPNmurA5ialfPzx3eOOOCg9iUB7rrJx+sg14G9YJymsGF0tkhxiGwRszh7F3AK3UBnrIQkxSyJqGPl+odcm3LQzD8SbK1IEticLWgKM0aDo7mgOYtWMLoLm4k0XFLIvOJUE5mC2h44WyaQTbLUG2yizvzvQA9lkEcBrOiSaaNwfE9ICzuBxqdomYrUKhmtNXQiV4nkq8vMdyyeALpRFBcJFElXCpQK7XTK9WYlmc7chNn40LdY4SQMno8UwsaAvHbz7WPDZdpgEX3Oca7NvFKxhgXWUKmbFZQCahUd1/h6dF2THBssTG5YxB92fWyngX0wlzjPY98BlscOq9atgxK8RazIiJKySNKVyAJOYKaxIkuTHOk7HqUgc3kXvT1TqeO3LGWlLhWLDJRjw0wVPfuMM0HjtdhpoeqoE3UauFB8G9+txaXrepotuIz+nDfAE6T8W5s3utSxQBNIHGtDJ0A1em4Cl8skrCuWSS5wKVVsAVbMq2WKa1YwTVKIhUInYdgUcyAENdBcKTrc6DLcNYCOnfHnoMZOxxObTWwLJElFmXQDoKNbojK+ZiQVW2aBeVxPhrsRVMV/s8rMDeWNDjbqZOoWrBvxmoTJEhRu+pk4sYWBBxtbGpSZcyRFRxdrbT/MyCpWnf8ANL0vWfRAQElVsVRYmEHP3RiQN0kAx396XSeyjx97U0AjAFZymWFdRhwY0AWtcGyoimKeQr9TlAAdXNfRRMDjE9lCjtvCtQNZqjwBxvEdmTqBb1g733CCfABNYycCuIHHCYwTTT/HZ4BQg9ZoUBOYUpotEwheXcb2smlFUtwDRfLoirRWJNQVNwJgE2/EVhdmpaN31VwTEbtQIjgt9qlAXZPO2Ns3SxOfrRzQ+BG9ZqZnbLjbMnq31iqfJ8lDrxRP8BZWmqosZ7Gr3pOMragjQynIYIg2uIjtDV5nTGiDFwk0gzVTJoUavi5FgtZNRqpKxHSzhtqiBTqKnldGoveVQIOlm+yRhMPyPmHOcnShaM4MusAq990MNbR/D6PjJmclpNLYhFAAA0P0EfQ3IJKjUKlOkw/BRDrKXRUll1s6GCx4kH4LWUVr6v1AHrM0dD4jmHem6JLeowL3Gy20sVixrPrDQJIjyZmG4Qz16v7ooYES0lVZSmXQsPEoQpsVNogZVCq6GGOFJ6TlPmYIRYjw3upoUEBM+M7uI32hOROpJ/J3ULWrdfHUyMglNSuqZu339UpWgxcNIUHXVDXjiIxEJVaaorfUYJgI7u4qbkjw4o1c6lc3ruz1Jbr0I77OkFkFphRBM+D31I8+BrQFekfN78wIqsPnPGTqJMRbwMju5hbB4m6zmmJFVjMmWBA/mLk7QX/tnviEWRiQDPGK40rArN9lBXNc6ybu4QbuvX7te/aUvh13s6emCbefXzxi7NuDyCLWND2s8yosiz7QewO3YsxdMMU06hGB1A6uewcTqgUfmXgJ3XMTjgOH/rmaGqTo54pqs6dp9/HZyo/vle9UBhjL41Z1ErvvkWryTnfdKftwchhBbGzn79ChXf8c3HnM2f+H5xvaxa4va6EAa4d5A6yGeEm8j2Rh+7jMsabIpWs32KDBrWpOyf/iNPiKZhR8g7lUrn19kIwIYY00pTDuDO+fV6Ww0JhMMN530GHaLS1A7W2ZhlQKJqDtQ7qkqmBO3ZgK6XZJN5iDrRmnS4o4XVOOsNZsKdzBtfP6w6wPLZlPKL9h/T2cPj/JpGeLWSXY54r2xyTi8OXr4Htcx8TjpqDUGg3L3YUkUggKuRVow8xqTFAgFKgMaTR2RY8qL3q0aWHJCfKkeaK4XDKCObIYjJg+gMVpsYOlRsY0no525Wqrw+h10tk2spfVGvuBx5xhna1kcpvAGXGNuQazVNqhRlYqdkfwhPsBIHdpLLbwpvlBLIRTrGbnXEtriO/ct0sIlqNf/S9m6Fxsm/8bQDdgy2thEM5nRBZlZagKi+Ekbny7sXTm2Vf9s4AZizsHwszfqtd/+u7P1va97BxHTbGvgmh7Ps3iRswe6rjBW6rQPzc+Of3KowHIhW997Pqf9DwvWpx3uH7veRyZvHxItn3dH5hi15mhd799uLJ7p4o65wn4S3OmiaIlFmRrtUqvnvF+LggCCp2hD29/RtfCfP/6DF2/u7z6z5/Rx2thfvoBvdistkhQZlZUIbKS2o9Kk0pRYuBb3/30v/7by6+DFKFmlVDG9ekBMnVW4PA4Hp2Y+x55zW8dL17XSIWveP68kO7KpgOYH9kw7sEPfAjfnmLaWiefmDIV5ujN+bsgsn9IQdP5so7jjP8jBZ2FaWvR/WJEKGzksPCEI3iOb/Cec1hiQzf4BCPSgbtv0HmeK/DTOi4PodM8vaQoj41zPjUWcn3x9sa9SqPhsQLrCaMfO04lp6n6txtd31hURrxfloZHToKIQkO79jgNa00sc9O1phUQHXRxnjP7ZczbgG1nln/4nZuQAaxJCBdc+ht+ucsCA1TaXOsket1DnzSM3nkMb6QyjUgeCN0cAmxwAMxsD0tePTHt3X6YWNaPSb2tt2OEFzRkN07lxfXYgeWLtZaEWZXT+Y0GOg6ycllhsaSzxnQiUizYslI0R/MtwKQih6yhsJwpj2w9MCgaHdGWg4suEvQ74BF1/24JV3QHgKKFNDTzmd3x84zikzYXOsOZS8VPALo0Kg3wRQKWWCSoFuYprkOq/idlAqLiPKs9cenU8r4Fb/cx66/WdSacQIO9MiuqBDXow7akZ+hj/Yy9AQfY9+imdoANXoLfxjS1elTPBMrEiGlcI+394mcIcx5UJsr2i5DghhUk5q2psm8gE0YibeAxZwJ9vB4VKAQSZJPJq+gi2wKVZYKxbxawojp2Rq8Fm6DExb2IsVPRwd+eAFs3WiHjVCyjT4oEnK3ykVALHdFAncqDeScAIxCBdIIFwugXqTZY5cM53QidLyHZSyFsb/w95NLNqdlQKsKqZ+SuiY+NcUuDeTdU55BB0DIeMiMGO2TC57lCWkLBjBVLfsRGeItrjsUUcfwHOCjrBJGOi3KwwV2XZRtJWVsLdgkG7O7LEztSSQl0IVjH6wf3sIg9VoaRimOFoF80qpF4cXX/8xu5lItFePo7JZlZ0eTHu4PsB7ugu40dvK8s3hbd88qsqDA+WXwUbV3F7JzwsIQet+Q46h81VaMIy8oQOS2l/ZLjCN9WhFCtR3CGzuPHNUc7LvEE8EJWxV1KtUWBwoQBblMIpx0caQ9HK5UgwKdLKey7YuVWSDlsfogGitLurtbx+tGNvJsYua6lUDPAGc2b/Xg/TE8fZgJpZqqA/ERQXEC9iPZQV1gjnMvSvi5mRZlCciPaI3OEM/heClmM5NXCTA7NXIv6aZUIq9wzkVv5I5VuCIDRL4xTdO4Rmw3I8BBnr2g25u7kaMJ4s/+TpCuMkuDWZy3EpUJojwFCxKx3fwIhXL7era/XiE2J8YTQuUxZPRDY/Jyu8JrJCrRLIotSyYKNZCjSqZG7EnjOoYhsgS7248bEuhE7CZHsY7ijdaIgAjsYRh0ucwSCgfUb/FKfbueVbe/bKNu1ZZaVMP1yttgafQ5l4Bk5xqx/kBYE7/GSCqoYqbcEBIFEv35qATMreGpDs92QR3ZGvptpo8aDn/Wejmm7dbI9vd6/J69euLUS7itomjZGuGEF1VauO21P0ZKOBpH8KURrCnHwIKDx4BOPQT2QtY7p3X0y1vr+YXv6LtPRhpw+eGveYXxoh4O9wY5bgfAAYfDl7u71wd2pSc/OXbQoe1OHTy5aL9VpBMgBOd4IkC+XHb8/fGSxRhtMc2QPk49qUgkS8449QH5Myo4x9zZgxkaphxK0np86euVOZVZZQc1KniBKgnc8ycih4b82euDQS0nJpF6nPVGd95J7f61FZA9fJvKE/Ofsxz/9Cb14c3l+8xJdMm2YWFZMr2gOpfBBXLhcyuR9gfZFwiBbduHw8McMXxzJGFMysVdxX/2nPdUQBs2NAY98tKHPj7kuBNL+m7rfjuMPcArFTLEItUlvM8Uwj9WdrreR9zhnlXYrIKmQZgXjWDnxZMWmvUME3vVweRXcc83yKTuNdDPlP1pGqL2Ivb6Y7SVPV2dxLvbddQhr+ErDjv/XO4ngkwEveMcN7ZRl5GFXplQpEwMGIRsgtVRLLNgfe7KqRTpWeCixj6B0l6dGyL1gKlhLmqjrzy92OXgtXIsv17toJ6v5V4q5WRGsKCoVzWXBBA4W3HXE0w02jAqjD6bHczzlbt/gk27WtX6kZSLGtVfnayu4SqwMNENqt7pfrE7Y7MgLm4dI1AXNqcKG5lm0pLI9/GGFzy/1ik3w7EbJNcub5mH+e7gsuddUB4zhm//YZ21Xpw0rOO0mWT7RLpslfa8/sx3ZZnB4KGROrpmLnq/6ivtIC7hG6Yw5FPyxmie9B52p86NOJfQysFGno4LGijXSRion8S20ghoMq30N35rZb30d3n3B8pzT6aTcW1jvoXIucLwduXeUnKvHY0yz3Ru/WqfDkNjW0dkzVHJsj8y+z1IhKojalmNefkiFnMCefEAGnWpsy1+lNugtJismRky6HCeSHF/1af1RQKZ/qagVH1Y/ck3O9Ay9yXGJPsH/OP0ol8LVnf5t+HiiFV5TqzlxihX6XFG1RdCDUJdSaFprVOHiVLvfDH4zjbz0PfCIhaxY3QVSuO27vnzjeNZbmgDVloHe++aoD8UUpjyldZj1ebxuLb3TxMjahv7hZRqpSoigHavPmpfHRZ5dG6mRGjsPMfMWZvqDwGjDRC43GumSErZgxH5yFqoT9Hmywwtit+fwbXNu0AvoCEsFaZ8hCF2+7FALVQLe8Td0ickWfdS7jW+bCGzRL6SNnl1rV5jAYB957bumFqACtWrAZPZFHFC86QMQqP7fqTSFcp4h+Xa3nV6hHuvO69TrwI5hh0FG8785YrPT5PWObdVn+HrXey3rrmDr411Ah7uZxmHXBAx2z6ZNyHTHMDihcEOKw8XPUDYQcyTgaIUbbDmnCya8rx6EE3T1K3A50nQQsDuqUCwRbq0Dpqf+xRaMjc829d59L6WR3pSND9sYTFbFxC3w21WB4GhgHXWPI8mQlzkT8SaIRb0bdstQVJj28QwIqW7ZDhyLa6PdlvcHpnYOsE779h3AusSq5in757N2K5sVG7RSR/Z2WFvWJb8/aHsm+swS19ZCqm26A/+LLrH414MdY2pEdruo1+p56GmyZPnLK4B+YG8nU4kGu6r7re/f1SgXZFQYJctjREcuq/nAufAgHvdrWmubHihHABxddce09/BCFiUW2+Y+wrWDcfrOXllTZZ+hjImFDCsFWN+lrhE6ID96VmSN2Yam7Yq++JwqR+CXivMt+o8Kc7ZgNEeXUPfsnINBVDZ0nhEp79iJgu6/0zly67f2M+Zj2nz0brNtOLysDKjcR44wPXzX3zdL+Ck73h3tfPIz9GFbuq23ngNLHHeC44en6CKL2ky2h7bFwTki1Nc61La2j8wUrrpGudzFznkWS6lqbz+EmN+/GTnyTq+cyOxU06JMO4doDynsygc99zWaSspEmsguUnYdex6oxCbsmiQiwzpmtL8DWPly+siQK8UjHnMHasRTaYzRrFKxvCEdmJqqDC/j2ZQt6OjP0y7oqOmPu6A91ycQLPTeUAGqVXzjxMKPxs2NordStJcqE1ujcktMUUu4I3M/wLKgXr3y/33hUXjl/8PnNYXc/phTFc7O89s5YfTcbaYbPAePa2fU2mA7uR+IZk0qJhZUqZG463Dfk+yrq/gfJH3QPTsBknVf4kXnGAJXCsLaMumVCiwxGftdubi9ZbsPkEGsun/6Kx0maI0P/GTliqpp/BFWZ/cZTy8uYPTjS3QB64dRo8pM1CxlhM4XVPnhn3QnC3NPc16aNHTcIWTnwO2iX+tOp+i9J83+ONYr+fjWKOHTRrfsj7C3ht0lkinXf71Cgi6lYe4AyxXWIxOgNJm6rVDnKN3i48MF7VEnmwA1SHDp8VjdOL2uvwknpGi2nKKiYre/UTP18MPooGUrTZjWVXSlEyBDslQ6b93TYiiAIVUqqQ90cChd6XllF0e3EJzeJ50myZBoOoP7KPKLW0jt3P8YdaTncUg+XnruwXFchGrNs3XKF70fUvWO7CAyeWZZD1fR2zTqVIDZHfUWdaLmBl+140q6DxLI1h+QhnidVOj69vyvb2/QjX2n0G9iZPpKi22iSupjsP2wkWFsQQyRFSV3+ign8sOEcNoeZKGhc02/zqZFGKSB+hGErRTco+VSxQZNIU+g5Do8mq4go0YD4GywqSab8NnFco05yx0jBpDoC8LJulrvE4RAsTu61X2xHYnz6wTSyLBXxpQ6YzCDNgloOMoUBCH4GdwmthR15YtUzGwP3CgiiyJpn7gH4u3w8A6hcAn+hinK+5ZmbBfLhmORaX2qgbd2ZSfDf/e7rWu0gti6UuOslGyKtOoQwg4DBBgAUmFrAMhKVliIQeOM1O2m/KqAyEjMdqK2zc3D4mce/v7m/J1/9171lm8eFCNV3/cfvWcb03fZWvIqFQHO6znOws+5aSZj1+N8K8GMRi8cEvoldOuAwt56om4PPAKkg7vhVSJp9sbj+lEw49MFZrtFB2uqIFNgUXFEpCC0NNZQvnVnONJeYbNJKX0d4a3BXo/QtoiWUhkkLX1//bfzUApukOyx+U6q5fQJlv0Cgx0X6xy7ZifBRjH/fvXbzfUNeovvCybyZqx3+Fjt3iZPw9wZojiyLb+Nwe72batRn8Ili9HTs12VY7aYrmDz1EX49ZaTqx07zjIvla8vfZdej8VeDPl0h3LiXgH1jov/8nXDTWGOyIeaZOzbDf4Sa0KfKLvRj6sGK74J6hauuPcM6SqQoo41+os2Sorlv845JnecaUPzv7zyfztrPmViQUn4owVTdIN5UJHBc975DcIiR1qiEbZUdMm0UVtr2U8pLEpsVr5Zf4MD6uMwQBKcUlOh6QqhXb0WkarThbzRJxvMqTCdnJQabz+QcdZMU5v1Lv847mN453SBK24yuBM/owXmO6XIO1vazeB/10mOqCdFtiPj27I1o/BiwQgMEphTKpCcQ9+ITkOv5lw0fsRm+hf7wFaGt75xGVusRWJ1stCp2ySNSBSFN6igWuOl70tEpJXfMMAspEi+kUt0SYnMR8I+HlZ0H5Xr+RwxgamH8JTSCIow7YsmF4gJbbAwNRphG9+wox7xfPhOBVVxuIfMWrfG1Tm14wnQytq2MGH3d2YE1bo+/cNTEARdU9VtUFFipSl6Sw0GTd3X3DZLvXgjl/rVjUuqfTkAf+nTwVq1AqP31AkLx+Gig+ZIJxm6TuLCeVq0udDLtMqzP+O3/p5fX37nAy6u7VtrXUNPgHtMDOJy6c5r2NcGdgeTrD23wPf07twh+3t/sLNRzhiAPopTQpwxgDzOKaNHsp72TF7/40z2n4ldNc2BPO36yvnfs2Cvq2eD3TpVqPRpqCmaMiv26WRLdf+fhhnYfukK7p+GHK5yZjLoR/0c0ds1nJ4RYquIE3WjIsbEcYil1ZhqyfF8OS2nRw2LTUu2BaV56iKQ8bBFt22iayRJ84EeMlASnmZF9PSQAfQDVsQ4FaevM+8Pxg2Sz5FrsM1I5EMBCj6afGQKtdpHBxo1WjX793/a7hq1F1IQ+zhgI5+7ZTsibqBJXUJx2KXuhV3GJb907vMbufRjXX0VA/SSsyaIol5QDba+YPc0R5rCpN2dH++uoccNlvoQBrCfbLA0hzAA/ahDGXoC4/uXjmPMwb4eQZPH0SBii4U9fPlrnVfqOZL3OVJT0XQe5nKpQ2zT8SF9OfRlxzDY4EejhL2+Wf/Q9gMcue594g52b+SXStz1T6nJ+9P/u+RNXPvkadyXC86R1vWW5QijJVtT0TjJvlxFwJLoOP9FWgskf47K35cR0Rh1aMhymyn6OcFZd4OHcMCwb9/M78r3FLuBi3TmvdkGuwprgocSZE7r5NGP18J89xOSCv3CJTbfv95N8yJSLNiyUuP5Le2+j1F3v+B9Qxj0uZZNgmU8Qc+MseyYuproS3cwSLXBKk+m1O2fVO8Ukk87+h5GinI8TE1zrVX9I+rR9s0wgVN12+VDKrZkAvP6N7vaygE6pNK/9iRGXN98+ilAAhTsJosikKDBaEjlGK9Py6hDxfHY12dFcZ6wvH7HtIOl0PXlU6KkDt9usBTAHBcrfdZONk6y5H423OTgtooWXBRrulxIzqFv6pcogC31TpBzY3mOaUQc6erxcB1F9Y0cjrMYJ/QztPgKMn8uqmohtakL9+bbwaE1k7gsQM2Kkm/9OdkvQzIzxWSFNMspevEnZFaqQq9//PEl2mA/SqheZQ8lnoXy+gBK+Lk6yUhBvhiucENVap9C03fVXmUdhIBe4Llc0w4xWLhEpxZv2iiKi9H7Q74YtjkxqWjOjmqacIhQX4U0x8axwBaImbrvD4j0V65NaI30cJzV3xDUi2ypQq/RlSC41BXHTbOyR8n1EPQnBj8CuZWhVb5/jf7FbvcMff89+hdEpLL6sus5UA9T++/c/E/7RabRLlHC7S+EzOmztXXFhmYEcz7H5C596VNOhTT1aDSwKywR65oXME3GptIBcyRvZgQsAw23MQeM3Rx7I5XVrMXWaR32g04zihBSCC1kJXL7wnAYyKChI8DDkhd3b8QAcoxYoL8Oe8JGI6ew5RLnz+Wd8+ggzf6AYZSKkYDV4U3h7pfBFnbPfS2E7bOPTavRykV9bDP0q9zYoxnanEwgqawxZiS6o7Q8QLRn8eJ9IURzgymydcqB51e15IGxVG4+tYBJ/B27cM0UjEy9vtz1vYuAi6M70x2I4Xbhr/r1JVJWWmtwqAxni4xO/28okaye+eSU2J1HMpIvlyQUNBT8bfOr99ANv5nRTBTFfhDQiKC0/9SBmC8g8OJXynTJWeruJc/WnNcsVSHsE1Okj2sa9VB+h1tn34B6IpDnutpq8U/If40IoxMvg3FBk8ToYQSQVOjm4vzG674EC0seVpRS9TVeBE/kF5cGUT0P98dH91SBIR4adYuGpnzV/qQ12J2eA5b5DL3+8Se0AboXFAuEOQ/7Curq5wVq/UdoQxV1YLFBnGJtkBS9cpFdIp5cTfyyiRi4qynCtp52v0uVA+Egq4mSlZBcLrf9QNyCqYEWi9CPiKywwsQ4IlJoX2SxcBPcUSV8Tg/f8ZmPVtTGLuh2gfqUQYR90xasRVFYJVOKOoyg8GZUpoFk7amVmIDG6mIUwvscJCGVqiFqg0WOVY6EVAXm7I9Qfq9URZA+uc9yOJpED5uFt4dILdYNMq84W1DYccDA15RIkY8o2O1xZ9pM0NA+tCEmiCxKTk2QAUadqBgU+PFG09pgZU7EyLd27SA7j7HyLmeOsl8hRfROyPkgQeLJTQ9EfiLCX4k8BdktyD+kOFH3nHr1WsV06bUf+hQeiKhkN/ocwTBuP4Lct8Otscv35YEFzvepzLbtjwJ/OkhFiVQ5zdO9gz7Jxj9Tulmx1jHqTJvmi934+vC1UrKYAdQKivI1oQIrJp1aX1TcsG8NowrhsuR19Uvby6bAAi9DpbkIcQjv1PaiQ8rhqhEzX2skN8JFxgwuyr5n0GNcT00a3j6jEVkxa93InOoZeltpA2ZSF6jrnjWSl4sNPfKQ9gqwxcLivaZTaEJwyPWCjnZuaJogjiGwVa1ztma51WyAH8KC7LYWZB96xAtv8r5karIdtufpYkH3lhOZ4Vu3WW2FntXXLFLAoPt9oxEP/UC371qezQZLtt3VqtgSqIg+irOhf+yrAhrk54pWk7GS5W7HRa183GAYe1p1G3B10SwBuVijHhqiRlQKdgiaQKYtC5Pg9V0WKXAtswSollkK7bmMKYp2gcYa9dFCTaArdV6R05iQPfMx+MYMnstHvTnHis1Dcu2YYEH7QPS6IcR2BGEyUOJjKNa64idqmi8rQ2RBXzkcGuPFD3AZcAgWngQ7BuQIg9A1Vcykbg061n3ar+6LAMdGk/ZcPhMPbnOvdFPpYqFB3MmNum8Nn7B264I5Yz1VvK6cPpspcACNi5Hlg8mwzSTYIN6hKTIJD+HTrpXetQSlQr/d+tRYpuuEgL5fDdavT2isSlKXUrOIguNBvAXmtMjb7sLN3R3twlNxk6VrXfRIUSSqgipGHiuLgnubaPLzAyrZmpvhxJK734OtranIYU7yQbkl538/QfeaOrQrh9Npu4ilrwUfkBvmAe9FzEn6lL3qvhqdBOvFjPdyrXCTWyykQbiZpBZOoOVymdWJKicR6jUjPlqoT9EzZUf2/TukW0HX6mHb70bxl5yR7RTTdkbkwg0g4JtrC74dkcsVT5k3HSbg+8o3/w+LUykMvU+tsTYIXbejAurqqjzX9l/wqGJeIxRqAHPgcSYrLJY0E3STWhaMBS7pphPqByXEGMXmlaEdCTHM0dcOdautd5+/kaHEJY4m7BrK8cGEjkluDhiC/fwih0xXfwsYt1ABZglWNxzUbc6XWlM1Q7fUHUqlqZrhJYVW3j7TfSFVjcMAdg3G6e0Efo/c7zt9K6RCcyU39rP6r6Se42jNrtF+0tf5DVYmtpuuARzbo+LvlBxUh051pyTP2xmkia6ULKkPKKZ6i88Fwpwq02QXqXZR/zcX3vLio9MEAJKQAgpzjoQU3ypaUrBk9mU/TDEXZbePfmgaitPjXjEXYavDP4Od+aEaraxHl7DgHKpNBJLi26W0/73nJQAlJQsojgn3jTvBwFeAgEVSLhBMmGdUz9BtK1P6gw26lVVpML5w5XyVtkaMKxl1yTa5F7/NNBPCK21qhvT/Mzgm+AnT9iR9TbT3b1jFFz4dV4Em137cDQtb9K4tUzql7OtDhpfF8hKwQFhrSRj4S+1pBO1JOLA37I7+3BlkCIMLz1CpYCbKGaKGfB1WlLHCsQZWHwhiwVLUUKVRiTV08dLQyMFPk5ZFYaWY3AnaD0trqCF71T33HpxK4+ucYYKHyYlvIouyGt7BBMeG0YaJXG58Pq2fNnnWZFKMEmOwzUXF+RZ9rjB3zs9cFpj5Qbyw73ohLkeerq7XM9EA+8FoOCbuaO5rgepEdKzBO+UNFPvJVw1qM5bvOzg+6AqRVNR1Jzs5t0QfgRq9325Phddvpfe8otthu54m6ExVwfqDnVK7WP2anTF5+zXt7yNr2gvG09/xZsu/wGrNNVY0rwhFdeSIht1tbqZ+FnhNkz0itztj/PvvY+cBtC/MqF+Akjt9VMuBGB5jv7p96FZYr5obatXCQJVhRVYu87eusWnKDC9qSL0WYXYjzTIzrYj9VfP/w0pTZOW5QAxy7ipBOMXK/gka4bWo+QLCevJrXdh5OPrghF817PP0rF8sIot5M753sfNg+bJR9YjXa81Upaf29HW1EUBg3OM3TYA0cCUu3OquJ+O4p9RZcNMNrnVe5utLP4IbvfCNG+rZlK7o1+L2MqxXOwf0qQb8e/fz9WV3vmsjJobeg92InEsDdFuYOSaysmDDdNhIXettyl72u1FdX6Dt1IW9fmzhjO+Jxx1fNAuj68uDmmws/9wBTdYi9lrkrUY7QxeuPtP3O+Xug/3aLCCodr/x3VfeHTevTFO5KU3zGFWCU+0oI92DspFojRXDcz6oAnRNGZhAJccjgkBToZP2R9k50K6q6laeWUllNYy6vpDZc759dX3T16GRbxnrPApjddlHDhR8cC1kG2lxSKJrYdAtWwoMwmKERUupUjav/XogvyyT3tS6m4SujvCfFpHOXQYuy2WAcd799gExQXiVUyvO/CBb+/MZenF1j4uS05/RjXOIOLAgvWdhvwhE5iaPbYJzqn1awpgxfWdV7iPwekQpXseN+c4/De+ZvtsTcjWKLZdUpRthFybZp24swOMA2ulKUb2SPLfc42z1kUmjO6H3CTwLw9i7l8ov3jsd42XTjOP6MlxG8uDoPJFFmU2cdwWn4nOvYIyr8+/pav6tRUcKqE9dwLgZmVdkzErzaumJssa6mDfSUiroPGDleo3fyJQ4rPINVqfJ0Bt21bfSFfuHyG5ipDXyCytEMXqLSd1POazcWhE0qR0jxbe1gqr2SyFna0Yfaq0o1tFzg7XBpoqlODf+KMz4ycwOu/hc3iOWvxp/v+zLWk2BocXo46DxsbsLFovw1a3fscTT9wZMfjmcu3fMc8aErGLFODt1JHoZ/U5ZSRrT6TDwyP4QGXDqzow7LHHOuZV7SFeEUK0XFUdXdn1EZE61ZYm62W/YsmAip/eRCcCZNsdpnk+ULbAwmGKqRmJOFcQ3C6wYhwyegAfPxd/FEmEg4rf2t8GdiQR8KOeuudCJNGK/OnrR5HOWVOnSF906CTMgmVcR2oT4usPTy5EiQ+fmGr7HqRNKnPLVJHl5X5X7tv0QM6FRTg1mPOBkmMvKdH43sjXJJ8/NrD22uMljAzzGH1JDi5Iny+Y5RzldYB8C8p0v6xi+z9a0WvGaKo63UMhlpH9c0YvAjbQfgNXtf00XdRW489Vrw0wFjRlRcGOtbTBs2PTU6xo1itXx7xAcG9MEsorIorD3KQ0bXTjoiHWSfUsl1yx3/rO6i1xB9WgiVC7J8YHGx3vLfmG81RpJNy8vrBrcl5D0dBpZX6+eVtb/Xc6P9Dsdvb3/Lec+ABO+XSVL1zj3EhKK3cnf3lyj64FC1UUjWddaX12yH4OIhV1NNewyqiH9GH+Yz60OK/dORGRzmaeu+BpU3PWVDo8LsriMqEer+N0SXMhggsrzjgvYlw67BNomHsKWLG9COSNOvCK21TgoA4/w8sdT8pp9l1XKZ6qe7n3z0XXPqQNRkKxxT0nV9SK41K85DZW31l2Y9iVuTOAICXrF812HSFNdideYcTwMZKDGFY6gvnJBlRqZtODu0DG+/nhxN2+sFL4BlAvADrbk0w00W85GJCIrsnmV59vo/hlWZFHrgDpwK02Pa3S+10sVH6JiMmKXg16JXaarKQoSmO5mr7qeq7jKmWkq69q+aB6j0GC7tmLDiZI2vLB/ky5LLDYF15NZ5RefrtALXyvxqeJWV54zDgUckAd2dV9Kbb/5En07dDSIfhTmTsiN2DGENCUVNLNY70IfmbRJ8AQuuH5a6EVd5f7Olya9oUtMtujjqLnG2VzhUxTl+4V3SMwEKjATC4ULujcdo8QKpvam75Owo1zewLLoncxdcnTbFrCTdRZACh3QviBVwBIilYW02zfuHd2gXysBpuRbmVOOXjCxnn1zhpgkZ2hu/0Xtv7DAfKuZnn0Tji8aUmYLjgeT82PrULsa/sUNgkXB1wVyclsPv5KLvY0ajEyKqfvr3ONZt0HQVFlGDiK0LuLK3R5mn97+jhVFH1wC8DfffHr7+/n7q2++cTm3a6wwG+XJjVR3MUuWD16w3+sFuxG2UScYFrGVCF+zE7dLSfMcYGKfi20CE2YhFRWakZgCpONKSoBxEd8LEogPxAKabTAbDid+sncAep/HBmqvT+wSdV3NE10KM8+1UbEr36FeO5lDrPuWRntH65qPdE7SY4td2sFgA5XGF5u0dS++3sWCWLBRR1O91WSO2GO3GuxGFNhmv7wnLJSP7if4eMeFRd7r/++Hq7Yqs5v8dxIWyzs+eo/IXiRPwhx1HHcfflJOkLS1c7Idu/SFaTLa6yw76JP5EtxuA849HJmuW1azKeJhUPS1wIxbWtfNXG68zLi+7Na2QScuaw4augy0MBjPKqxzrjOrIh6xn2MSryHd2lcfXciiqETfEzXAThzXuOmp2L2j9+bfaVinbnDTx2nWT8XtFov832Q4atbiZrBhx0iGJ2M3XHgHOV3pkhEmo2WJTmXBA/YbrMQw6PDcUdeiKDOZShjfvnt7g35zftQ2KTWMyOdJUwlu/+MN+lxRNdK7teIiU7TfqTNtckPHIbpF7+uis2BaV6Olk4gPaReojD1GwAItj3IcHYJqAsGxJ8PN4w9owByrIsFpWbAJ3Au4jFiA3ACt8mhTaXdgxu12tQM6x6avFT4V7pwKsiqwilVW0sDdlngwvvjJ0SdMBulUUWBmq+i8QOgibgFVA3ixhFZLCcDK+d8TQC1x9EkYruNUdPaCoHvGYj84vnNbQa3qGR1pkWECg1Hil59Y2FpENN47gOfLcv2DuDer6O87ERkxKst11L7rHegW8nGRpwcAXnMcXWKIjIolExGLIoegU+RGi2yR6Q0zJLr8ENmCy43GRfzclS5sYdbpoCeIuhCRMZFSnDBRUlXMt9ES3gewS3KXBvga8xS8wsqsVNLILH5ICqCvf8jA4xgfNk92N7lcZnkKYlvA8fPfiMgKfJ8ZE8ttsAvYcjSnCR6FgolESDORDumS64zPeRY7LLoD+08JgUfvDN6BHbsXYhd27KreLuwfE8L+KSHsf04I+38khP3nNLCNLDme0xQipYEe3zwTWVFxUL7n2wTvZA28vEuglxQVZ8uiTKN9Wy0T82XsJCQPmaVQSjT9TOL7RkSmXUJighPUiqSxJi3gNNak3uqqTDCLlIimrDqJqWqksaYHvU8gQow01jBLBRvMmiTAK8HuBRZSU5KACdc/WaokehTWP8nSrCjOE7jVZFFmhCfwYVvACYIkAFfNtya+W9RC1kkgl1WWIKZBFDOMYJ6ggEhneEkF2UbMuurCFphv/6D5PAXe6wzagCaB7NrBpMHaJdYmgT5fluuf0vigdTZn5s9JGo0RncWdFdcDrGR0Ua2TXHOASomKX+WmnY8/2qytDmBqVs7PH9854oCD2pcEuOsmH6+DXAf2gnGawobR2SLFIbJFzOLsXcApdAOdsRKSFLMkoo6V6x9ybcpBM/9IsLUiSWBztqApzBgNjuaC5ixawegubCbScEkh84pTTWQKanvgbJlANslSb7CJOvO/Az2UQR4FsKJLpo3C8T0hLewEGp+iZSpSq2S01tCJXCWSry4z37F4AuhGUVwkUCRdKVAqtNMp15uVZDpzE2bjQ99ihZMweD5SCBsD8trNt48Nl2mDRfQ5x7k280rFGhZYQ6VuVlAKqFV0XOPr0XVNcmywMLlhEX/Y9bGdBvbBXOI8j30HWB47rFq3DkrwFrEiI0rKIklXIgs4gZnGiixNcqTveJSCzOVd9PZMpY7fspSVulQsMlCODTNV9OwzzgSN12KnhaqjTtRp4ELxbXy3Fpeu62m24DL6c94AT5Dyb23e6FLHAk0gcawNnQDV6LkJXC6TsK5YJrnApVSxBVgxr5YprlnBNEkhFgqdhGFTzIEQ1EBzpehwo8tw1wA6dsafgxo7HU9sNrEtkCQVZdINgI5uicr4mpFUbJkF5nE9Ge5GUBX/zSozN5Q3Otiok6lbsG7EaxImS1C46WfixBYGHmxsaVBmzpEUHV2stf0wI6tYdf4D0PS+ZNEDASVVxVJhYQY9d2NA3iQBHP/pdZ3IPn7sTQGNAFjJZYZ1GXFgQBe0wrGhKop5Cv1OUQJ0cF1HEwGPT2QLOW4L1w5kqfIEGMd3ZOoEvmHtfMMJ8gE0jZ0I4AYeJzBONP0cnwFCDVqjQU1gSmm2TCB4dRnby6YVSXEPFMmjK9JakVBX3AiATbwRW12YlY7eVXNNROxCieC02KcCdU06Y2/fLE18tnJA40f0mpmeseFuy+jdWqt8niQPvVI8wVtYaaqynMWuek8ytqKODKUggyHa4CK2N3idMaENXiTQDNZMmRRq+LoUCVo3GakqEdPNGmqLFugoel4Zid5XAg2WbrJHEg7L+4Q5y9GFojkz6AKr3Hcz1ND+PYyOm5yVkEpjE0IBDAzRR9DfgEiOQqU6TT4EE+kod1WUXG7pYLDgQfotZBWtqfcDeczS0PmMYN6Zokt6jwrcb7TQxmLFsuoPA0mOJGcahjPUq/ujhwZKSFdlKZVBw8ajCG1W2CBmUKnoYowVnpCW+5ghFCHCe6ujQQEx4Tu7j/SF5kyknsjfQdWu1sVTIyOX1KyomrXf1ytZDV40hARdU9WMIzISlVhpit5Sg2EiuLuruCHBizdyqV/duLLXl+jSj/g6Q2YVmFIEzYDfUz/6GNAW6B01vzMjqA6f85CpkxBvASO7m1sEi7vNaooVWc2YYEH8YObuBP21e+ITZmFAMsQrjisBs36XFcxxrZu4hxu49/q179lT+nbczZ6aJtx+fvGIsW8PIotY0/SwzquwLPpA7w3cijF3wRTTqEcEUju47h1MqBZ8ZOIldM9NOA4c+udqapCinyuqzZ6m3cdnKz++V75TGWAsj1vVSey+R6rJO911p+zDyWEEsbGdv0OHdv1zcOcxZ/8fnm9oF7u+rIUCrB3mDbAa4iXxPpKF7eMyx5oil67dYIMGt6o5Jf+L0+ArmlHwDeZSufb1QTIihDXSlMK4M7x/XpXCQmMywXjfQYdpt7QAtbdlGlIpmIC2D+mSqoI5dWMqpNsl3WAOtmacLinidE05wlqzpXAH187rD7M+tGQ+ofyG9fdw+vwkk54tZpVgnyvaH5OIw5evg+9xHROPm4JSazQsdxeSSCEo5FagDTOrMUGBUKAypNHYFT2qvOjRpoUlJ8iT5onicskI5shiMGL6ABanxQ6WGhnTeDralautDqPXSWfbyF5Wa+wHHnOGdbaSyW0CZ8Q15hrMUmmHGlmp2B3BE+4HgNylsdjCm+YHsRBOsZqdcy2tIb5z3y4hWI5+9b+YoXOxbf5vAN2ALa+FQTifEVmUlaEqLIaTuPHtxtKZZ1/1zwJmLO4cCDN/q17/6bs/W9v3snMcNcW+CqLt+TSLGzF7qOMGb6lC/9z45PQrjwYgF771set/0vO8aHHe4fq953Fk8vIh2fZ1f2CKXWeG3v324crunSrqnCfgL82ZJoqWWJCt1Sq9esb7uSAIKHSGPrz9GV0L8/3rM3T97vLqP39GH6+F+ekH9GKz2iJBmVlRhchKaj8qTSpFiYFvfffT//pvL78OUoSaVUIZ16cHyNRZgcPjeHRi7nvkNb91vHhdIxW+4vnzQrormw5gfmTDuAc/8CF8e4ppa518YspUmKM35++CyP4hBU3nyzqOM/6PFHQWpq1F94sRobCRw8ITjuA5vsF7zmGJDd3gE4xIB+6+Qed5rsBP67g8hE7z9JKiPDbO+dRYyPXF2xv3Ko2GxwqsJ4x+7DiVnKbq3250fWNRGfF+WRoeOQkiCg3t2uM0rDWxzE3XmlZAdNDFec7slzFvA7adWf7hd25CBrAmIVxw6W/45S4LDFBpc62T6HUPfdIweucxvJHKNCJ5IHRzCLDBATCzPSx59cS0d/thYlk/JvW23o4RXtCQ3TiVF9djB5Yv1loSZlVO5zca6DjIymWFxZLOGtOJSLFgy0rRHM23AJOKHLKGwnKmPLL1wKBodERbDi66SNDvgEfU/bslXNEdAIoW0tDMZ3bHzzOKT9pc6AxnLhU/AejSqDTAFwlYYpGgWpinuA6p+p+UCYiK86z2xKVTy/sWvN3HrL9a15lwAg32yqyoEtSgD9uSnqGP9TP2Bhxg36Ob2gE2eAl+G9PU6lE9EygTI6ZxjbT3i58hzHlQmSjbL0KCG1aQmLemyr6BTBiJtIHHnAn08XpUoBBIkE0mr6KLbAtUlgnGvlnAiurYGb0WbIISF/cixk5FB397AmzdaIWMU7GMPikScLbKR0ItdEQDdSoP5p0AjEAE0gkWCKNfpNpglQ/ndCN0voRkL4WwvfH3kEs3p2ZDqQirnpG7Jj42xi0N5t1QnUMGQct4yIwY7JAJn+cKaQkFM1Ys+REb4S2uORZTxPEf4KCsE0Q6LsrBBnddlm0kZW0t2CUYsLsvT+xIJSXQhWAdrx/cwyL2WBlGKo4Vgn7RqEbixdX9z2/kUi4W4envlGRmRZMf7w6yH+yC7jZ28L6yeFt0zyuzosL4ZPFRtHUVs3PCwxJ63JLjqH/UVI0iLCtD5LSU9kuOI3xbEUK1HsEZOo8f1xztuMQTwAtZFXcp1RYFChMGuE0hnHZwpD0crVSCAJ8upbDvipVbIeWw+SEaKEq7u1rH60c38m5i5LqWQs0AZzRv9uP9MD19mAmkmakC8hNBcQH1ItpDXWGNcC5L+7qYFWUKyY1oj8wRzuB7KWQxklcLMzk0cy3qp1UirHLPRG7lj1S6IQBGvzBO0blHbDYgw0OcvaLZmLuTownjzf5Pkq4wSoJbn7UQlwqhPQYIEbPe/QmEcPl6t75eIzYlxhNC5zJl9UBg83O6wmsmK9AuiSxKJQs2kqFIp0buSuA5hyKyBbrYjxsT60bsJESyj+GO1omCCOxgGHW4zBEIBtZv8Et9up1Xtr1vo2zXlllWwvTL2WJr9DmUgWfkGLP+QVoQvMdLKqhipN4SEAQS/fqpBcys4KkNzXZDHtkZ+W6mjRoPftZ7Oqbt1sn29Hr/nrx64dZKuK+gadoY4YYVVFu57rQ9RUs6GkTypxCtKcTBg4DGg088BvVA1jqmd/fJWOv7h+3pu0xHG3L64K15h/GhHQ72BjtuBcIDhMGXu7vXB3enJj07d9Gi7E0dPrlovVSnESAH5HgjQL5cdvz+8JHFGm0wzZE9TD6qSSVIzDv2APkxKTvG3NuAGRulHkrQen7q6JU7lVllBTUreYIoCd7xJCOHhv/a6IFDLyUlk3qd9kR13kvu/bUWkT18mcgT8p+zH//0J/TizeX5zUt0ybRhYlkxvaI5lMIHceFyKZP3BdoXCYNs2YXDwx8zfHEkY0zJxF7FffWf9lRDGDQ3Bjzy0YY+P+a6EEj7b+p+O44/wCkUM8Ui1Ca9zRTDPFZ3ut5G3uOcVdqtgKRCmhWMY+XEkxWb9g4ReNfD5VVwzzXLp+w00s2U/2gZofYi9vpitpc8XZ3Fudh31yGs4SsNO/5f7ySCTwa84B03tFOWkYddmVKlTAwYhGyA1FItsWB/7MmqFulY4aHEPoLSXZ4aIfeCqWAtaaKuP7/Y5eC1cC2+XO+inazmXynmZkWwoqhUNJcFEzhYcNcRTzfYMCqMPpgez/GUu32DT7pZ1/qRlokY116dr63gKrEy0Ayp3ep+sTphsyMvbB4iURc0pwobmmfRksr28IcVPr/UKzbBsxsl1yxvmof57+Gy5F5THTCGb/5jn7VdnTas4LSbZPlEu2yW9L3+zHZkm8HhoZA5uWYuer7qK+4jLeAapTPmUPDHap70HnSmzo86ldDLwEadjgoaK9ZIG6mcxLfQCmowrPY1fGtmv/V1ePcFy3NOp5Nyb2G9h8q5wPF25N5Rcq4ejzHNdm/8ap0OQ2JbR2fPUMmxPTL7PkuFqCBqW455+SEVcgJ78gEZdKqxLX+V2qC3mKyYGDHpcpxIcnzVp/VHAZn+paJWfFj9yDU50zP0Jscl+gT/4/SjXApXd/q34eOJVnhNrebEKVboc0XVFkEPQl1KoWmtUYWLU+1+M/jNNPLS98AjFrJidRdI4bbv+vKN41lvaQJUWwZ675ujPhRTmPKU1mHW5/G6tfROEyNrG/qHl2mkKiGCdqw+a14eF3l2baRGauw8xMxbmOkPAqMNE7ncaKRLStiCEfvJWahO0OfJDi+I3Z7Dt825QS+gIywVpH2GIHT5skMtVAl4x9/QJSZb9FHvNr5tIrBFv5A2enatXWECg33kte+aWoAK1KoBk9kXcUDxpg9AoPp/p9IUynmG5NvddnqFeqw7r1OvAzuGHQYZzf/miM1Ok9c7tlWf4etd77Wsu4Ktj3cBHe5mGoddEzDYPZs2IdMdw+CEwg0pDhc/Q9lAzJGAoxVusOWcLpjwvnoQTtDVr8DlSNNBwO6oQrFEuLUOmJ76F1swNj7b1Hv3vZRGelM2PmxjMFkVE7fAb1cFgqOBddQ9jiRDXuZMxJsgFvVu2C1DUWHaxzMgpLplO3Asro12W94fmNo5wDrt23cA6xKrmqfsn8/arWxWbNBKHdnbYW1Zl/z+oO2Z6DNLXFsLqbbpDvwvusTiXw92jKkR2e2iXqvnoafJkuUvrwD6gb2dTCUa7Krut75/V6NckFFhlCyPER25rOYD58KDeNyvaa1teqAcAXB01R3T3sMLWZRYbJv7CNcOxuk7e2VNlX2GMiYWMqwUYH2XukbogPzoWZE1Zhuativ64nOqHIFfKs636D8qzNmC0RxdQt2zcw4GUdnQeUakvGMnCrr/TufIrd/az5iPafPRu8224fCyMqByHznC9PBdf98s4afseHe088nP0Idt6bbeeg4scdwJjh+eoossajPZHtoWB+eIUF/rUNvaPjJTuOoa5XIXO+dZLKWqvf0QYn7/ZuTIO71yIrNTTYsy7RyiPaSwKx/03NdoKikTaSK7SNl17HmgEpuwa5KIDOuY0f4OYOXL6SNDrhSPeMwdqBFPpTFGs0rF8oZ0YGqqMryMZ1O2oKM/T7ugo6Y/7oL2XJ9AsNB7QwWoVvGNEws/Gjc3it5K0V6qTGyNyi0xRS3hjsz9AMuCevXK//eFR+GV/w+f1xRy+2NOVTg7z2/nhNFzt5lu8Bw8rp1Ra4Pt5H4gmjWpmFhQpUbirsN9T7KvruJ/kPRB9+wESNZ9iRedYwhcKQhry6RXKrDEZOx35eL2lu0+QAax6v7pr3SYoDU+8JOVK6qm8UdYnd1nPL24gNGPL9EFrB9GjSozUbOUETpfUOWHf9KdLMw9zXlp0tBxh5CdA7eLfq07naL3njT741iv5ONbo4RPG92yP8LeGnaXSKZc//UKCbqUhrkDLFdYj0yA0mTqtkKdo3SLjw8XtEedbALUIMGlx2N14/S6/iackKLZcoqKit3+Rs3Uww+jg5atNGFaV9GVToAMyVLpvHVPi6EAhlSppD7QwaF0peeVXRzdQnB6n3SaJEOi6Qzuo8gvbiG1c/9j1JGexyH5eOm5B8dxEao1z9YpX/R+SNU7soPI5JllPVxFb9OoUwFmd9Rb1ImaG3zVjivpPkggW39AGuJ1UqHr2/O/vr1BN/adQr+JkekrLbaJKqmPwfbDRoaxBTFEVpTc6aOcyA8Twml7kIWGzjX9OpsWYZAG6kcQtlJwj5ZLFRs0hTyBkuvwaLqCjBoNgLPBpppswmcXyzXmLHeMGECiLwgn62q9TxACxe7oVvfFdiTOrxNII8NeGVPqjMEM2iSg4ShTEITgZ3Cb2FLUlS9SMbM9cKOILIqkfeIeiLfDwzuEwiX4G6Yo71uasV0sG45FpvWpBt7alZ0M/93vtq7RCmLrSo2zUrIp0qpDCDsMEGAASIWtASArWWEhBo0zUreb8qsCIiMx24naNjcPi595+Pub83f+3XvVW755UIxUfd9/9J5tTN9la8mrVAQ4r+c4Cz/nppmMXY/zrQQzGr1wSOiX0K0DCnvribo98AiQDu6GV4mk2RuP60fBjE8XmO0WHaypgkyBRcURkYLQ0lhD+dad4Uh7hc0mpfR1hLcGez1C2yJaSmWQtPT99d/OQym4QbLH5jupltMnWPYLDHZcrHPsmp0EG8X8+9VvN9c36C2+L5jIm7He4WO1e5s8DXNniOLItvw2Brvbt61GfQqXLEZPz3ZVjtliuoLNUxfh11tOrnbsOMu8VL6+9F16PRZ7MeTTHcqJewXUOy7+y9cNN4U5Ih9qkrFvN/hLrAl9ouxGP64arPgmqFu44t4zpKtAijrW6C/aKCmW/zrnmNxxpg3N//LK/+2s+ZSJBSXhjxZM0Q3mQUUGz3nnNwiLHGmJRthS0SXTRm2tZT+lsCixWflm/Q0OqI/DAElwSk2FpiuEdvVaRKpOF/JGn2wwp8Ko7T/93wAAAP//Gm+GWA==" + return "eJzs/W1zHDeSII6/30+Bc8T/LDno1li2tTfe2b3gktKaN5LMFSV5/xcTUYFGobsxRAElANXN9qf/BRKoZ1STbAJFam/8wmGzuxOJRCKRz/k9uqb7XxBhmsh/Qsgww+kv6Mz/b041Uaw0TIpf0L/9E0IIvZN5xSlaSYU2WOScibX7OhLU7KS6RjndMkIRl2u9+CeEVozyXP/yT/Br+8/3SOCC+jUXWOPmE4TMvqS/oLWSVdn5awCN+p83AB3QcVicXp2iN0zRHeZ80flqjUb7lxqPgmqN1zRjeQ+yQ+Wa7ndS9T85gA5CHze0g4mHjVhOhWErRlWLUwAVXa1W7OaOaNAbXJT2tDTVmklxdxx/g79j7tdDeGWoQv8/i/BdEZWVIjRjwlC1woTGoNwVwEQNTDhUs6FoxeUOSYXolgpzEK2casMEtvDj4nbeAn4QgqriNLP/GQOp97igSK4AhVNCqNboTAqjJEdvmTawGDIbbFCBDdnQHJkN03fA0p9upalKgauF6/BiGv7g1vPkvBOG3YOeDc3OovfBtcBlSfOsvjJlAM/BH28VMEZhoTk2NK9pd3GJcJ4rqvU9cNlIbe5MtRWuuMlAjP6CVphr+lCc7fL3wLaUKoQtl2L9UEws6Ltg0pMvkQ+yy133O80uVo91pF3s73quXbxTHG4Xp1tP2GwUxSbjdEt5HD3AwkMAD6RFgfkOK4peoKU0ghqL6WrFyAL9JkDmbKnaf8/l7gTZfw3AFTKnCht6gjZsvbGPDXzd/s9dtkWwoWup9jF2duZhNc/f9M7e2EexVlO2TFX6xH9nuD+j5N+xOEHUkIP7IVIIStwFjKKvfRLsS9VV0GBbGN70g5gwUpSZXTSAhd4M2fkgDhdn7y7hl7cvSGQea0EL6q60nthnGrHy+fJ9Z23UWzukC+AyU5RIlev7IfIADR9rzdaC5uj89BINFw+SsiiwyDPOBM2wWlcFFWY+dP3yyC6PmuWtibamOVru4RpzSTBHuMqZsZ8c2k69/eEjeMc93PeVbJ/DlvBGIuw4hTMqDNIVaMCrivN9wz3i4C5KxbaM0zVdSH5PHj7yLH7fUIEwaJa6Xd7ql2SDxbrW0L2+KXmOtphXB7m/3YSguye4CUF3t29iWSltFnL5d0qGUizdpVDUqQluWRD7gAfaYSWYWB+80A5jNg/bdLG1SgC6OD8KXVIpRYXJLIz5ZI9b1CML6GtKxR2wlWLF1pWi+eMg3K7fwf12tPF2/Tj44i1VeE0fROhHQ75D7MA+MOdyR/O7cHhRcWzYlmZEVmI+YWKkwRzBmlaX7+C+YUYjzQShTqg7abPDGhGrmlsBpBDhFKvOBvs+0pXpYvNwH+kbpmgpd1TVVso5XVGh6RNxnL75eP51OU4twv9wnP7DcfoPx+lX7ThFnzRFr8+u/EcLgc2Clf/wpx7pTw2R8wk7Wht0O5/fgwX+4YS9Hac+Wwzp/A8X7T9ctP9w0fYWvNVFqympFDMhpgl6U9oFB8t9wDuv6QNxrzxc9Nq+0oejUF+5mzgligfdxH0bj0kd1ca7+O2qScGp/5k25TBowRln93i47qgN2ifW6dgW+kFOWmHCeJibD9pxV6/P7ncu9ULISLTbMLJxQtLbnIquqNLo2aoVjSfo6v27yxN09f+/OkFYWEVnAHYlldk8X6DTFjjBAi0pwmiDVQ7i16VGnSCMSiWNJJKfIBBlhcuqkquhzLVK/l4bWiAtV8YCWaALg3IqpKE9I8BLeoIr3dDe/XT4TrltLkaM6BO4Fo2dthhYB3JL1U4xYx8tVdERv44P6ZYrdOCguixUZ5a1BuRuQ5Xzp/iHDG2wRktKBZJLTdWW5uP9qV6q2W2bGV++g1uZvluAtcB9lWV69an1Q0t0tDm9HhzzoRUO3arBqXy0tto13VtbrtIu8EJwaSpPf4V3zcUBm4/Igmq7aWk/H4BG6K1co3NqHzYV3oiDxYZIHbudGi6Ym1bbJZEBe4QTU9+T3N14IoWBAJ5cISa0wcLUaOggjoYVxyCYDz3BIey8jW+XQNh4cYpr35pzf2L0nprfmRH2GfCnvxixRrNZvZEVz5GgW6qsBK35rsRKU/SOGmxRw2ilZNFZ6tlbudYvLjG5pkY/H4E/Z4oSw/cnTXwKow/UCQvH4aKD5iJIyLHtcTdKjkyoASXPaakoAYPJYpLTFbNqgxQc0DJ4ya0SX4axKvR6qGrH5UB/xu/8Pb84/8HF9LyXp1bM3bfoDSYQQXbnpUYHAbtjoLQ5boHv2eMosTKMVBwr+L0/2MUkZ4xAH8UpIc4YQZ7mlMkj2c57Ji//cSaHz8SumuZAHnZ95fLvGWxkeCxPBrstPkboJUdNUaf7PkXcLNlS3f+HYaYNNrSgg+DoE0EO0o8ywvHgDj8R9KgwAw/dE0FsM/I6PRHEmDgOsbQaUy05ni6n5RQfIz3Skm1FXcQglg01odeE7MzOF2u3gMVmpIeMlISHWREDPWQE/RYrYpqKo8DrLFQUHa9KkHyOXKNtRiIfClDw3uQjc6jV1SjmUO/f/2nfN2rPpCD2ccBGPnXLdkLcbFlacdil7pldhq0Ywd37/FauXbyhzmipRE4VOEupF1Sjra/YDc2RppB11ftxfw09bbDUhzCC/WCDpTmEEeh7HcrYExjfv3QcY472dQ+a3I8Go5B6Er78VWrTFZF8yJGaipyJdf2hDrFNx4f09dCXHcNgox9NEvbicvtTk8M/dd2HxB3t3sivlbjbV6nJ++r/XfKOos5JZMNQLjhHWtdbliOM1mxLReMk+3oVAUui4/wXaS2Q/Ckqf19HRGPSoSHLfabolwRn3Q0ewgHDvn292Wu3NLqEi3TivdkGo4/7kiKCxxJkSRFlZkMV+nQhzA+vkFToDZfY/PgSLbEGLqoDZFBNAKrfLfs+Rt39ivcNYdB0xmcE/0IwEW4W67he+at3MEi1w2pUnRlN6+hItM62u5S8uPzc0/cw1K8NjxTVuS3uEfVoQ7o9dZyqHfGgcEaxNYPaC/ebvrZyCx1S6V8HEiMuLj+/CpAgnJODIpCgwWhM5RivT8uoY8Xx2NdnQ3FO1Syx619hKXRx/pAoqcO3GywFMMfFSp+0k42TLLmfDdeK1kWraMFFsabLmeScEiPV1yiALfUeIefG8hzTiDjS0dxi2lNU38qh2oIOEPoJWnwFWT4VVbWQGpLdCinQcj86NIQU/VJRDUVQmhUl3/tzsl+GRF2KyQZpllP07E/IbFSFXv7883MoDdWUimaVA5R4EsrrHSihSyk0TUcK8tVwhSsRrn0KVbF0Qs9eZR2EgJ7hpdzSDjGYCGZW1uJNG0VxMXl/yFfDNo9MKpqzaqinxSDUNyHNsXEssBVi5m/Vyz/98GftRPqLEgRojfTfRrv5m7UH3+I9Veglei0ILjUUwUsBJuW95HoI+gODH4HcytAqP75E/2q3e4J+/BH9KyJSQcsLOCa36An6n9z8i/0i06hPlG+CRyhkHigafiK2rtjRjGDOl5hcp9WAHXJ1wQA2zq6wRKQiLyUTpu4uEkQUmCOjSslE+WmtPqhLShjmgDFgqo1UVrMWe6d12A+2mLPcMUYIKYRWshK5fWE4BeSZWHvl6Nbkxf6NGEGOEQv01+FA2GjiFPZc4vypvHMeHaTZHxQV1ChGAlaHN4W7XwZb2D33tRC2zz42rUYrV/WxLdCvcmePZmxzMoGkssaYkeia0vIWoj2JF+8rIZqSUAy2ZXmWp4q6vq4lz5oKqJrVUFZVOTva24VbpkyFuTXae753EXBxsIJZsxti5UAMtwt/1S/OkbLSWoNDBYiG1Zqa5mu3UkKrRElPj06Jumb/ECVUklDQWPBfnNe+1w+0kIaiK8/vda+c5X5KUCLoNuICMV9B4MWvlOmSs5SZDU/anNdspPY/Cd3MytyE/A63zr4BdZmm57raavFPyH+PCKMTLyvGHyFGb1e1xtHl2eml1319US4rSqmGGi+CJ/KrS4Oonob7w7dpAEN83HwNOVdq35Sv2p+0BrvTc8AyX6CXP79CO6B7QbFAmPOwrwCc+qAmtf4jtKPK9cBD0OYDa4OkGJSL9In46Gri103EwF1NEbb1tPtdqhwIB1lNlGyE5HK9HwbiVkyNtFiEfkZkgxUmxhHRXuo94A9Oc4Eq4XN6eM9nPllRG7ug2wXqUwYRDsQuwaIorJIpRR1GUHg3KdNAsg7USkxAY3UxCuF9DpIQ6PEIELXBIscqR0KqAnP2Ryi/V6oiSJ/cZzkcTSJZLUdP0r2I1GLdIPOCsxWFHQcMfE2JFPmEgt0ed6ZNSj/LgQ0xQWRRcmqCDDDpRMWgwBvFBmKwU2+mzCMx8pVdO8jOU6zc58xJ9iukMJtIx9TWp8bKeWmznPJHIvxrkacguwX5hxSpuy0cEIt29VrFdOm1H4cUHomoZDf6FBl6Y/zlQ1uqdKecIj+UBxY434cy257iWNtsy/SIVDnN072DPsnGP1O6WbHWMepMm+aL3fj6+LVSslgA1AqK8jWhAismnVpfVNyw7w2jCuGy5HX1S9vLpsACr0OluQhxCO/02vrUDaUQM99qJHfCRcYMLsqhZ9BjDP03lRwnHzGjEdkwa93InOoFeldpA2ZSF6i9ldhM5OViQ488pIMCbLWyeG/pHJoQHHK9oKMdtIKigjiGwFa1ztmW5VazAX4IC7KrWpB9HBAvvMmbkqnZdtiep4sF3VhOZIbv675XRoK+ZpFyzRkP+kYjHvqkC+fESuNGni1GSzbpZLKKLYGKkSL3UIgN/WNfFdAgv1S0mo2VLHc7Lmrl4w5rBEjkE3wDyP0Qm6gRlYIeQRPItHVhEry+6yIFrmWWANUyS6E9lzFFUR/oy+hQE+hKnVfkcUzIgfkYfGNGz+W93pxjxeZtcu2YYEH7QAy6IcR2BGEyUuJjKNa64qnDThNWlKwMkQV94XBojBfIyh41wESWLxwJegbkBIPQLR21w51tY/XqvgiwE9k55PJJW7w46h3oXumm0sVCg7hTSQlbsdbwCWu3vpX7BE95XTl9NlPgABoXI8vbgonaRZX7IEsQb282z3UIn/tWetcSlAr9duVTY5muEwKGfjXk+8IOBiigXpWkLqVmEQXHnXgLzGmRuw5TkMpf393JLjwVN+OG2Y8likRVUMXIfWVRcG8zVLEd2Fi3kq25GU4sufs92tqWilwqnzB7cGdy+fdH6F5Th3YDbc27iKWvBR+R20rQw4g5SZ+yV9034wvpq/69mPFerg1ucouFNAjDmAiLZDiBlst1VieqPIpQrxnx3kJ9jp4pPdn3H5BuBV2r++MOu1iVkjOyT317DsiFS0DAN9cWfD8hl4PDlhIT8EPFKSAWFqdSGHqTWmNtELoQzl/X9kPFea7tv+BRhVFvgFCoAcwtj7ObkpkNx3UmkAVTgct6JGfTKwQbo9iyMrQjIcY5+n7Ap9XWu89fWHTocjhB7OFWixv1Ov/NAUNwmF/k58529LeAcQsVYJZgdcNB3eZ8qS1VC3RF3aFUmqoFXlNo5e0z3VdS1TiMYNdgnN5O3NAt9/tO3wqp0FLJnf2s/qvXNZ3ZNdlP+iK/xMrEdtM1gGN7VPydGs7xne9ONbN6E14pWVIfUEz1Fp8KhDlVpskuUu2i/m8uvOXFR6cJACQhBRTmHAkpvle0pGDJHMp+ALNhzienHj7a2CumGdD5grkIWx3+Ge1sx8zGK8tO1qNzWHAJ1SYCSfH9Wtr/PvASgJKSBRTHhPvGnWDgC0DAIilXyEoHw6heoKtWpgwHG3Qrq9JgfObK+SptjRhXMuqSbXIvfj3hMSK80qZmSP8/o2OCnzBtT9LXRHv/hlV84dNpFWh27cfdsLBF79oypVPKvr3N8LJYngMWCGstCQN/qT2NoD0JB/aWXdNfEEblZq8ZwRzlTF+foFLBTBQYJfZtWFHGCh9Te3nPh97V2ShcUAPDzLGGLl4aGjm4XgT16HzZC9qPS2t6U9HQ+Gly78FjaXydM0zwMDnxTWRRVuM7mODYMNoxkcudz6clUhBampMmk2KSGKNtrirO9+hLhblzfuaywEx4qSE6C3E58XR1vZ6x1KUDW7cq4Vsmrmnua4HqRHSswTvlDRT7yTcNaguWHzo4PuoKkVTUdSc7ObfEEIEavd+uHguv30rveUVX43Y9TdCZqoINBzuldrH6NQFbx/+HNe0fI2vaK8bT3/Fmy29gteYaK5pXhKI6ckTD7jZNFcM8C7ymyR6RK1iyVpuH72PnAbQvzKRfgJJrfVTLgRgeY7+6feg2WG+aG2rVwkCVYUU2LvO3rrFpygzPakiDFmF2I80yC60IDL6v/39caYqsPBeIQc5dJWBEvv0TNMJrUfMFhO0QPFfYeXv0wQm/atzn6Um/WEQWy3qerlz1HixfNqru8XrBwNe5PX1dbQQQmPb4zRMgDVyJM7e668k47Sl1Flxy13hDPudlvjhH752keeYbNyA3bc8X/Vrcnof1aueAfgxffsf9fHEOJPUlb42YGHsP+hE5lwbotrBwTGRlwY7psJG61fuUvez7UV1foO3UhYN+7InhyIkv3Vk7Kffi/FZNNpZ/7hZN1iL2UuStRrtAZ64+0/c75e6Dw9osIKj63/jhG++OW1amqdyUpnmMKsGpdpSR7kHZSbTFiuElH1UBuqYMTKCS4wlBoKnQSfuj9A60q6q6lRdWUlkNo64vZPacr15cXA51aORbxjqPwlRd9pEDBe9cC9lGWhyS6EIYdMXWAoOwmGDRUqqUzWu/Hckvy6SXte4moasj/KdFpDt82nJZLgOM8/63j4gJwqucWnHmB9m6QfjPXtcDjC+dQ8SBBem9CPtFIDI3e2wTnFPt0xLGjOlrq3Ifgdc9SvE6bsz3/mn4wPT1gZCrUWy9pirdCLswyT53YwEeBzeiWVG9kTy33ONs9YlJo73Q+wyehXHs3UvlZx+cjvG8acZxcR4uI7lzdJ7IosxmzruCU/G5VzDG1fn3dLX83qIjBdSnrtxs7rwiU1aaV0sfKWusi3kjLaWCzgNWrtf4TUyJ84PIH0UBHHfVX8Hsc/cQ2U1MtEZ+ZoUoRu8wqfsph5VbK4JmtWOk+L5WUNVhKeRszehDrRXFOnpusDbYVLEU58YfhRl/NLPDLr6UN4jlL6bfL/uyVnNgaDH6NGp87O6CxSJ8det3LPH0vRGTn4/n7h3znDEhq1gxzk4diV5Hv1NWksZ0Oow8sj9FBpy6M2OPJU45t3IP6YoQqvWq4ui1XR8RmVNtWaJu9hu2LJjI6U1kAnCmzXGa5wNlCywMppiqkVhSBfHNAivGIYMn4MFz8XexRhiI+L39bXBnIgEfyqVrLvRIGrFfHT1r8jlLqnTpi26dhBmRzKsIbUJ83eHp+USRoXNzjd/j1AklTvlqkry8r8p9236ImdAopwYzHnAyLGVlOr+b2Jrks+dm1h5b3OSxAR7TD6mhRcmTZfOcopyusA8B+c6XdQzfZ2tarXhLFcd7KOQy0j+u6FngRtoPwOr2v6arugrc+eq1YaaCxowouLHWNhg3bHrodY0axer4dwiOjWkCWUVkUdj7lIaNzhx0xDrJvqWSW5Y7/1ndRa6gejIRKpfk+EDj/b1lbxhvtUbSzcsLqwY3JSQ9PY6sr1dPK+v/LpdH+p2O3t7/kUsfgAnfrpKla5x7DgnF7uSvLi/QxUih6qKRrGutry45jEHEwq6mGnYd1ZC+jz/M51aHlXsnIrKlzFNXfI0q7oZKh8cFWVwm1KNN/G4JLmQwQ+V5xwXsS4ddAm0TD2FrljehnAknXhHbahyVgUd4+eMpec2+yyrlM1VP97785Lrn1IEoSNa4oaTqehFc6teShspb6y5MhxI3ZnCEBL3ied8h0lRX4i1mHI8DGahxhSOor1xRpSYmLbg7dIyvP17czRsrhW8A5QKwoy35dAPN1osJiciKbFnl+T66f4YVWdQ6oA7cStPjGp0f9FLFh6iYjNjlYFBil+lqjoIEprvZq67nKq5yZprKurYvmscoNNiurdhwoqQNLxzepMsSi03B7WxW+dnn1+iZr5X4XHGrKy8ZhwIOyAN7fVNKbb/5HH0/djSIYRTmWsid6BlCmpIKmlls+9AnJm0SPIMLbpgWelZXub/3pUlv6RqTPfo0aa5xtlT4MYry/cI9EjOBCszESuGCHkzHKLGCqb3p+yT0lMtLWBa9l7lLjm7bAnayzgJIoVu0L0gVsIRIZSH1+8a9pzv0ayXAlHwnc8rRMya2i+9OEJPkBC3tv6j9FxaY7zXTi+/C8UVDymzF8Whyfmwdqq/hn10iWBR8XSAn9/XwK7k62KjByKSYur8uPZ51GwRNlWXkIELbIq7cHWD2+d3vWFH00SUAf/fd53e/n354/d13Lud2ixVmkzy5k+o6ZsnyrRfs93rBboRt0gmGRWwlwtfsxO1S0jwHmNjnYp/AhFlJRYVmJKYA6biSEmBcxPeCBOIDsYBmO8zGw4kf7B2A3uexgdrrE7tEXVfLRJfCLHNtVOzKd6jXTuYQ676l0d7RuuYjnZP02GKXdjDYSKXxxSZt3Yuvd7EgVmzS0VRvNZkj9titBrsRBbY5LO8JC+Wj+wne33Fhkff6/4fxqq3K7Cb/PQqL5R0fvUfkIJKPwhx1HPcQflLOkLTVO9mOXfrMNBntdZYd9Ml8Dm63EefeHpmuW1azOeJhUPS1woxbWtfNXC69zLg479a2QScuaw4aug60MJjOKqxzrjOrIh6xn2MSryHd2lcfncmiqMTQEzXCThzXuOmh2L2nN+Y/aFinbnDTx2nWD8XtCov832U4atbiZrBhx0iGB2M3XriHnK50yQiT0bJE57LgAfsdVmIcdHjqqGtRlJlMJYyv3r+7RL85P2qblBpG5MusqQRX//kWfamomujdWnGRKTrs1Jk2uaHjEN2jD3XRWTCtq9HSScSHtAtUxh4jYIGWRzmOboNqAsGxB8PN4w9owByrIsFpWbAJ3Au4jFiA3ACt8mhTaXsw43a76oHOsRlqhQ+Fu6SCbAqsYpWVNHD3JR6NL35w9AmTUTpVFJjZJjovELqKW0DVAF6todVSArBy+fcEUEscfRKG6zgVnb0g6J6x2A+O79xWUKt6RkdaZJjAYJT45ScWthYRjfcO4OW63P4kbswm+vtOREaMynIdte96B7qFfFzk6Q6AtxxHlxgio2LNRMSiyDHoFLnRIltlescMiS4/RLbicqdxET93pQtbmG066AmiLkRkTKQUJ0yUVBXLfbSE9xHsklynAb7FPAWvsDIrlTQyix+SAujbnzLwOMaHzZPdTS7XWZ6C2BZw/Pw3IrIC32TGxHIb9AFbjuY0waNQMJEIaSbSIV1ynfElz2KHRXuw/5QQePTO4B3YsXshdmHHrurtwv45IexXCWH/c0LY/ysh7D+ngW1kyfGSphApDfT45pnIioqD8r3cJ3gna+DldQK9pKg4WxdlGu3bapmYr2MnIXnILIVSoukXEt83IjLtEhITnKBWJI01aQGnsSb1XldlglmkRDRl1UlMVSONNT3oTQIRYqSxhlkq2GDWJAFeCXYjsJCakgRMuH1lqZLoUdi+kqXZUJwncKvJoswIT+DDtoATBEkArlruTXy3qIWsk0AuqyxBTIMoZhjBPEEBkc7wmgqyj5h11YUtMN//QfNlCry3GbQBTQLZtYNJg7VLrE0Cfbkut6/S+KB1tmTmz0kajRGdxZ0VNwCsZHRRrZNcc4BKiYpf5aadjz/arK0OYGo2zs8f3znigIPalwS46yYfr4NcB/aKcZrChtHZKsUhslXM4uw+4BS6gc5YCUmKWRJRx8rtT7k25aiZfyTYWpEksDlb0RRmjAZHc0FzFq1gtA+biTRcUsi84lQTmYLaHjhbJ5BNstQ7bKLO/O9AD2WQRwGs6Jppo3B8T0gLO4HGp2iZitQqGa01dCJXieSry8x3LJ4AulEUFwkUSVcKlArtdMr1biOZztyE2fjQ91jhJAyeTxTCxoC8dfPtY8Nl2mARfc5xrs2yUrGGBdZQqZsVlAJqFR3X+Hp0XZMcGyxMbljFH3Z9bKeBQzDXOM9j3wGWxw6r1q2DErxFrMiIkrJI0pXIAk5gprEiS5Mc6TsepSBzeR29PVOp47csZaUuFYsMlGPDTBU9+4wzQeO12Gmh6qgTdRq4UHwb363Fpet6mq24jP6cN8ATpPxbmze61LFAE0gca0MnQDV6bgKX6ySsK9ZJLnApVWwBViyrdYprVjBNUoiFQidh2BRzIAQ10FwpOtzoMtw1gI6d8eegxk7HE7tdbAskSUWZdAOgo1uiMr5mJBVbZ4F5XA+GuxNUxX+zyswN5Y0ONupk6hasG/GahMkSFG76mTixhYEHG1salJlzJEVHF2ttP8zIJlad/wg0vSlZ9EBASVWxVliYUc/dGJB3SQDHf3pdJ7JPnwZTQCMAVnKdYV1GHBjQBa1wbKiKYp5Cv1OUAB1c19FEwOMT2UKO28K1A1mqPAHG8R2ZOoFvWDvfcIJ8AE1jJwK4gccJjBNNv8RngFCD1mhQE5hSmq0TCF5dxvayaUVS3ANF8uiKtFYk1BU3AmATb8RWF2alo3fV3BIRu1AiOC32oUBdk87Y2zdrE5+tHND4Eb1mpmdsuPsyerfWKl8myUOvFE/wFlaaqixnsavek4ytqCNDKchgiDa4iO0N3mZMaINXCTSDLVMmhRq+LUWC1k1GqkrEdLOG2qIFOoqeVkaiD5VAo6Wb7JGEw/I+Y85ydKZozgw6wyr33Qw1tH8Po+MmZyWk0tSEUAADQ/QR9DcgkqNQqU6TD8FEOsq9Lkou93Q0WPBW+q1kFa2p9x15zNLQ+Yxg3pmia3qDCjxstNDGYsW6Gg4DSY4kZxqGM9Sr+6OHBkpIV2UplUHjxqMI7TbYIGZQqehqihUekJZ7nyEUIcJ7q6NBATHhO7tP9IXmTKSeyN9B1a7WxVMjI9fUbKhatN/XG1mNXjSEBN1S1YwjMhKVWGmK3lGDYSK4u6u4IcGzt3KtX1y6stfn6NyP+DpBZhOYUgTNgD9QP/oY0BboPTW/MyOoDp/zmKmTEG8FI7ubWwSLu81qihXZLJhgQfxg5u4M/bUH4hNmYUAyxAuOKwGzftcVzHGtm7iHG7gP+rUf2FP6dtzNnpom3H5+8YSxbw8ii1jTdLfOq7As+khvDNyKKXfBHNOoJwRSO7juPUyoFnxi4iV0z004Dhz652pqkKJfKqrNgabdx2cr379XvlMZYCyPW9VJ7KFHqsk77btTDuHkMILYWO/v0KFd/xLceczZ/7fPN7SLXZzXQgHWDvMGWA3xknjvycL2cVliTZFL126wQaNb1ZyS/8Xj4CuaUfAN5lK59vVBMiKENdKUwrgzfHhelcJCYzLDeN9Rh2m3tAC1t2UaUimYgHYI6ZKqgjl1Yy6k2yXdYA62ZZyuKeJ0SznCWrO1cAfXzusPsz60ZH5E+Q3rH+D05aNMeraYVYJ9qehwTCIOX74Ovsd1TDxuCkqt0bDcXUgihaCQW4F2zGymBAVCgcqQRmNX9KjyonubFpacIE+aJ4rLNSOYI4vBhOkDWDwudrDUxJjGx6NdudnrMHqddLadHGS1xn7gMWdYZxuZ3CZwRlxjrsEslXaokZWK3RE84X4AyF0aiy28aX4QC+EUq8Up19Ia4r37dg7BcvSr/8UCnYp9838j6AZseS0MwvmCyKKsDFVhMZzEjW83ls48+2Z4FjBjsXcgzPytevmnH/5sbd/zznHUFPsmiLbn0yxuxOyujhu8pwr9c+OT0y88GoBc+NbHrv9Jz/OixbnH9QfP48jk5dtk27fDgSl2nQV6/9vH13bvVFHnPAF/ac40UbTEguytVunVMz7MBUFAoRP08d0v6EKYH1+eoIv356//6xf06UKYVz+hZ7vNHgnKzIYqRDZS+1FpUilKDHzrh1f/+388/zZIEWo2CWXckB4gUxcFDo/j0Ym5757X/Mrx4kWNVPiK508L6a5sugXzIxvG3fmBD+E7UExb6+QzU6bCHL09fR9E9g8paDpf1nGc8X+loIswbS26X40IhY3cLjzhCJ7iG3zgHNbY0B1+hBHpwN2X6DTPFfhpHZeH0GmeXlKUx8Y5HxoLuTh7d+lepcnwWIH1jNGPnlPJaar+7UYXlxaVCe+XpeGRkyCi0NCuPU3DWhPL3HSteQVEB12c58x+GfM2YNuZ5R9+52ZkAGsSwgWX/oaf91lghEqba51Er7vrk4bRe4/hpVSmEckjoZtDgA0OgJn97ZJXz0x7tx8m1vVjUm/r3RThBQ3ZjXN5cT12YPlirSVhVuV0fqORjoOsXFZYrOmiMZ2IFCu2rhTN0XIPMKnIIWsoLGfKI1sPjIpGJ7Tl4KKrBP0OeETdv1vCFd0BoGghDc18Znf8PKP4pM2FznDmUvETgC6NSgN8lYAlVgmqhXmK65Cq/0mZgKg4z2pPXDq1fGjB230shqt1nQmPoMG+NhuqBDXo476kJ+hT/Yy9BQfYj+iydoCNXoLfpjS1elTPDMrEhGlcI+394icIcx5UJsr2i5DghhUk5m2psm8gE0YibeAxZwJ9upgUKAQSZJPJq+gi2wKVZYKxbxawojp2Rq8Fm6DExb2IsVPRwd+eAFs3WiHjVKyjT4oEnK3ykVALndBAncqDeScAIxCBdIIVwuiNVDus8vGcboRO15DspRC2N/4GcumW1OwoFWHVM3LXxPvGuKXBvBuqc8ggaBkPmRGjHTLh81whLaFgxoolP2IjvMUtx2KOOP4dHJR1gkjHRTnaYN9l2UZSttaCXYMB2395YkcqKYEuBNt4/eDuFrHHyjBScawQ9ItGNRLPXt/88lau5WoVnv5OSWY2NPnx9pD9aBd0t7GD92uLt0X3tDIbKoxPFp9EW1cxOyfcLaHHLTmN+idN1STCsjJEzktpv+Q0wlcVIVTrCZyh8/hxzdGOSzwBvJBVcddS7VGgMGGE2xzCqYcjHeBopRIE+HQphX1XrNwKKYfND9FIUervahuvH93Eu4mR61oKNQOc0bzZj/fDDPRhJpBmpgrITwTFBdSLaA91gzXCuSzt62I2lCkkd6I9Mkc4g2+kkMVEXi3M5NDMtaifV4mwyj0TuZU/UumGABi9YZyiU4/YYkSGuzh7RbMxdycnE8ab/T9KusIkCa581kJcKoT2GCBEzHr3BxDC5etd+XqN2JSYTghdypTVA4HNL+kGb5msQLsksiiVLNhEhiKdG7nXAi85FJGt0Nlh3JjYNmInIZJDDHtaJwoi0MMw6nCZIxAMrN/gl/p0O69se98m2a4ts6yEGZazxdbocygDz8gxZv2dtCB4j9dUUMVIvSUgCCT6DVMLmNnAUxua7YY8sgvyw0IbNR38rPd0TNutR9vTy8N78uqFWyvhvoKmaWOEG1ZQbeW60/YULelkEMmfQrSmELceBDQefOAxqDuy1jG9ux+NtX68255+yHS0Iad33pp3GN+2w9HeYMetQLiDMPh6d/fy1t2pWc/OXbQoe1O3n1y0XqrzCJBb5HgjQL5edvzx9iOLNdpgniO7m3xUs0qQmHfsDvJjVnaMubcRMzZKPZSgDfzU0St3KrPJCmo28hGiJLjnSUYODf+1yQOHXkpKJvU6HYjqfJDc+2stIgf4MpEn5L8WP//pT+jZ2/PTy+fonGnDxLpiekNzKIUP4sLlWibvC3QoEgbZsiuHhz9m+OJExpiSib2Kh+o/7amGMGhuDHjkow19vs91IZD239T9dhx/gFMoZopFqE16mymGeazudIONfMA5q7RbAUmFNCsYx8qJJys27R0i8K6Hy6vgnmuWz9lppJsp/8kyQu1FHPTFbC95ujqLU3HorkNYw1cadvy/3kkEn4x4wTtuaKcsIw+7MqVKmRgwCtkAqaVaY8H+OJBVLdKxwl2JfQSluzw1Qe4VU8Fa0kRdf97Y5eC1cC2+XO+iXlbzrxRzsyFYUVQqmsuCCRwsuOuIp0tsGBVG35oez/Gcu32LH3WzrvUjLRMxrr0631rBVWJloBlSu9XDYnXGZkde2NxFoq5oThU2NM+iJZUd4A8rfN7UKzbBs0sltyxvmof57+Gy5F5THTGGb/5jn7W+ThtWcNpNsnymXTZL+l5/Zj+xzeDwUMic3DIXPd8MFfeJFnCN0hlzKPh9NU96AzpT50edSuh1YKNORwWNFWukjVRO4ltoBTUYVvsWvrWw3/o2vPuC5Tmn80m5d7DeXeVc4Hg7cu8oOVePx5hnu5d+tU6HIbGvo7MnqOTYHpl9n6VCVBC1L6e8/JAKOYM9eYcMOtXYlr9KbdA7TDZMTJh0OU4kOb4Z0vqTgEz/UlErPqx+5Jqc6QV6m+MSfYb/cfpRLoWrO/3b+PFEG7ylVnPiFCv0paJqj6AHoS6l0LTWqMLFqXa/GfxmHnnpe+ARC1mxugukcNt3ffmm8ay3NAOqLQN98M1R74opTHlK6zAb8njdWrrXxMjahv7hZRqpSoigHatPmpfHRZ5dG6mJGjsPMfMWZvqDwGjHRC53GumSErZixH5yEqoT9Hmy4wtit+fwbXNu0DPoCEsFaZ8hCF0+71ALVQLe8bd0jckefdL9xrdNBLYYFtJGz661K8xgsE+89l1TC1CBWjVgMvsijije9AEIVP/3Kk2hnGdMvv620yvUU915nXod2DHsMMho/jdHbHaevN6prfoMX+96r2Xda9j6dBfQ8W7mcdg1AYP+2bQJme4YRicUbkhxe/EzlA3EHAk4WeEGW87pignvqwfhBF39ClxONB0E7I4qFEuEW+uAGah/sQVj47NNvXffS2miN2XjwzYGk00xcwv8dlUgOBpZR93jSDLkZclEvAliUe+G3TIUFaZ9PANCqlu2A8fi2mi35f2BqZ0jrNO+fbdgXWJV85T980m7ld2GjVqpI3s7rC3rkt/vtD0TfWaJa2sh1T7dgf9Fl1j8260dY2pE+l3Ua/U89DRZsvzlBUC/ZW+PphKNdlX3Wz+8q0kuyKgwSpbHiI5cVsuRc+FOPO7XtNY2vaUcAXB01R3z3sMzWZRY7Jv7CNcOxuk7e2VLlX2GMiZWMqwUYH2dukboFvkxsCJrzHY0bVf01ZdUOQJvKs736D8rzNmK0RydQ92zcw4GUdnRZUakvGaPFHT/nS6RW7+1nzGf0uajd5ttw+FlZUDlPnKE6e13/UOzhJ+y493Rzie/QB/3pdt66zmwxHEnOH14iq6yqM1kB2hbHJwjQn2rQ21rh8jM4aprlMs+ds6zWEpVe/shxPzh7cSRd3rlRGanmhZl2jlEB0hhV77Vc1+jqaRMpIn0kbLr2PNAJTZh1yQRGdYxo/0dwMqX00eGXCke8Zg7UCOeSmOMZpWK5Q3pwNRUZXgdz6ZsQUd/nvqgo6Y/9kF7rk8gWOiNoQJUq/jGiYUfjZsbRW+j6CBVJrZG5ZaYo5awJ3M/wrKgXr3w/33mUXjh/8PnNYXc/phTFc7O89t5xOi520w3eA4e186otdF2cj8QzZpUTKyoUhNx1/G+Z9lXV/G/lfRB9+wMSNZ9iVedYwhcKQhry6RXKrDEbOz32sXtLdt9hAxi1f3TX+k4QWt64CcrN1TN44+wOrvPeHp2BqMfn6MzWD+MGlVmpmYpE3Q+o8oP/6S9LMwDzXlp0tBxh5CdA7eLfqs7naIPnjT741iv5P1bo4RPG12xP8LeGnadSKZc/PU1EnQtDXMHWG6wnpgApcncbYU6R+kWnx4uaI862QSoUYLLgMfqxul1/U04IUWz9RwVFf3+Rs3Uw4+Tg5atNGFaV9GVToAMyVLpvHUPi6EAhlSppD7Q0aF0pedruzi6guD0Iek0S4ZE0xncR5GfXUFq5+HHqCM9j0Py/tLzAI7TIlRrnm1TvujDkKp3ZAeRyTPLeriK3qZRpwLMrqm3qBM1N/imHVfSfZBAtv6ENMTrpEIXV6d/fXeJLu07hX4TE9NXWmwTVVIfg+3HnQxjC2KIbCi51kc5ke8mhNP2IAsNnWv6dTYtwiAN1I8gbKXgAS2XKjZqCvkISq7Do+kKMmk0AM4Gm2q2CZ9dLLeYs9wxYgCJoSCcrav1IUEIFLumez0U25E4v04gjQx7Y0ypMwYzaJOAhqNMQRCCn8BtYmtRV75Ixcz+lhtFZFEk7RN3R7wdHt4hFC7B3zFF+dDSjO1i2XEsMq0fa+CtXdnJ8N/9busarSC2rtQ4KyWbI606hLDDAAEGgFTYGgCykg0WYtQ4I3W7Kb8qIDIRs52pbXPzsPiZh7+/PX3v370Xg+WbB8VINfT9R+/ZxvR1tpW8SkWA03qOs/BzbprJ2PU430owo9Ezh4R+Dt06oLC3nqg7AI8A6eBueJVImr31uH4SzPh0gUW/6GBLFWQKrCqOiBSElsYaylfuDCfaK+x2KaWvI7w12OsR2hbRUiqDpKXvr/9+GkrBDZI9Nt9JtZ4/wXJYYNBzsS6xa3YSbBTzH69/u7y4RO/wTcFE3oz1Dh+r3dvsaZi9IYoT2/LbGO3u0LYa9Slcshg9PdtVOWar+Qo2H7sIv95ycrWj5yzzUvni3Hfp9VgcxJDPdyiP3Cug3nHx375uuCnMEflYk4x9u8FfYk3oR8pu9OOqwYpvgrqFK+49QboKpKhjjf6ijZJi/W9Ljsk1Z9rQ/C8v/N9Omk+ZWFES/mjFFN1hHlRk8JJ3foOwyJGWaIItFV0zbdTeWvZzCosSm41v1t/ggIY4jJAEp9RcaLpCaFevRaTqdCFv9MkGcypMJyelxtsPZFw009QWg8s/jfsU3jld4YqbDO7EL2iFea8Uubelfgb/+05yRD0psh0Z35atGYVXK0ZgkMCSUoHkEvpGdBp6Neei8T02M7zYt2xlfOsbl7HFWiRWJwuduk3ShERReIcKqjVe+75ERFr5DQPMQorkW7lG55TIfCLs42FF91G5ns8RE5gGCM8pjaAI075ocoWY0AYLU6MRtvENO+oRz8fvVFAVh3vIrHVrXJ1TO54AbaxtCxN2f2dGUK3r0799CoKgW6q6DSpKrDRF76jBoKn7mttmqWdv5Vq/uHRJtc9H4M99OlirVmD0gTph4ThcdNCc6CRDt0lcOA+LNhd6nVZ59mf8zt/zi/MffMDFtX1rrWvoCXCDiUFcrt15jfvawO5gkrXnFvie7s8dsr/3B7uY5IwR6KM4JcQZI8jTnDJ5JNt5z+TlP87k8JnYVdMcyMOur1z+PQv2unoy2G1ThUofhpqiKbNiH062VPf/YZiB7Zeu4P5hyOEqZyaDftRPEb2+4fSEENtEnKgbFTEmjkMsrcZUS46ny2k5PWpYbFqyrSjNUxeBTIctum0TXSNJmo/0kJGS8DArYqCHjKDfYkVMU3H+OvPhYNwg+Ry5RtuMRD4UoOC9yUfmUKt9dKBRo1Wzf/+nfd+oPZOC2McBG/nULdsJcQNN6hKKwy51z+wyLvmlc5/fyrUf6+qrGKCXnDVBFPWCarT1FbuhOdIUJu32ftxfQ08bLPUhjGA/2GBpDmEE+l6HMvYExvcvHceYo33dgyb3o0HEFgsH+PLXOq/UcyQfcqSmouk8zOVah9im40P6eujLjmGw0Y8mCXtxuf2p7Qc4cd2HxB3t3sivlbjbV6nJ++r/XfImrn3yNB7KBedI63rLcoTRmm2paJxkX68iYEl0nP8irQWSP0Xl7+uIaEw6NGS5zxT9kuCsu8FDOGDYt2/m99r3FLuEi3TivdkGuwprgscSZEnr5NFPF8L88ApJhd5wic2PL/tpXkSKFVtXajq/pd33MeruV7xvCIM+1bJJsIxn6JkxlR1TVxN97Q4GqXZY5cmUusOT6p1C8rmn72GkKMfj1DTXWtU/oh5t3wwTOFW3XT6kYmsmMK9/09dWbqFDKv3rQGLExeXnVwESoGA3WRSBBA1GYyrHeH1aRh0rjse+PhuK84Tl9T3TDpZCF+cPiZI6fLvBUgBzXKz0STvZOMmS+9lwk4PbKlpwUazpciY5h76pX6MAttR7hJwby3NMI+JIV4+H6yiqb+V4nMU0oZ+gxVeQ5VNRVQupTV24t9yPDq2ZxGUBalaUfO/PyX4ZkpkpJhukWU7Rsz8hs1EVevnzz8/RDvtRQvUqByjxJJTXO1DCz9VJRgry1XCFG6pS+xSavqv2KusgBPQML+WWdojBwiU6tXjTRlFcTN4f8tWwzSOTiubsqKYJtxHqm5Dm2DgW2AoxU/f9AZH+wrUJrZEej7P6G4J6kT1V6CV6LQgudcVx06zsXnI9BP2BwY9AbmVolR9fon+12z1BP/6I/hURqay+7HoO1MPU/ic3/2K/yDTqEyXc/kLInD5ZW1fsaEYw50tMrtOXPuVUSFOPRgO7whKxrnkB02RqKh0wR/JmRsAy0HAbc8DYzbE3UlnNWuyd1mE/6DSjCCGF0EpWIrcvDIeBDBo6AtwtebF/I0aQY8QC/XU4EDaaOIU9lzh/Ku+cRwdp9gcMo1SMBKwObwp3vwy2sHvuayFsn31sWo1WrupjW6Bf5c4ezdjmZAJJZY0xI9E1peUtRHsSL95XQjQ3mCLbphx4/rqWPDCWys2nFjCJv2MXbpmCkakX533fuwi4OLoz3YEYbhf+ql+cI2WltQaHyni2yOT0/4YSyeqZH50S/XkkE/lySUJBY8HfNr/6AN3wmxnNRFHsBwFNCEr7Tx2I+QoCL36lTJecpe5e8mTNec1SFcI+MEX6uKZRd+V3uHX2DagnAnmuq60W/4T894gwOvEyGhc0S4weRgBJhS7PTi+97kuwsORhRSnVUONF8ER+dWkQ1dNwf3xyTxUY4qFRt2hsylftT1qD3ek5YJkv0MufX6Ed0L2gWCDMedhXUFc/r1DrP0I7qqgDiw3iFGuDpBiUi/SJ+Ohq4tdNxMBdTRG29bT7XaocCAdZTZRshORyvR8G4lZMjbRYhH5GZIMVJsYRkUL7IouFm+COKuFzenjPZz5ZURu7oNsF6lMGEQ5NW7AWRWGVTCnqMILCu0mZBpJ1oFZiAhqri1EI73OQhFSqhqgNFjlWORJSFZizP0L5vVIVQfrkPsvhaBLdbRbeASK1WDfIvOBsRWHHAQNfUyJFPqFgt8edaTNDQ/vQhpggsig5NUEGmHSiYlDgpxtNa4OVeSRGvrJrB9l5ipX7nDnJfoUU0Tsh56MEiQc3PRD5IxH+tchTkN2C/EOKR+qeU69eq5guvfbjkMIjEZXsRp8iGMbtR5D7drg1dvmhPLDA+T6U2fbDUeAPB6kokSqnebp30CfZ+GdKNyvWOkadadN8sRtfH79WShYLgFpBUb4mVGDFpFPri4ob9r1hVCFclryufml72RRY4HWoNBchDuGd2l50SDlcNWLmW43kTrjImMFFOfQMeozrqUnj22c0IhtmrRuZU71A7yptwEzqAnXdsybycrGhRx7SQQG2Wlm8t3QOTQgOuV7Q0c4NTRPEMQS2qnXOtiy3mg3wQ1iQXdWC7OOAeOFN3pRMzbbD9jxdLOjGciIzfO82q63Qs/qaRQoY9LBvNOKh39Ltu5Zni9GSbXe1KrYEKqKP4mzoH/uqgAb5paLVbKxkudtxUSsfdxjGnlbdBlxdNEtALtaoh4aoEZWCHkETyLR1YRK8vusiBa5llgDVMkuhPZcxRVEfaKxRHy3UBLpS5xV5HBNyYD4G35jRc3mvN+dYsXmbXDsmWNA+EINuCLEdQZiMlPgYirWu+CM1zZeVIbKgLxwOjfHiB7iMOAQLT4KeATnBIHRLFTOpW4NOdZ/2q/siwKnRpAOXz8yD29wr3VS6WGgQd3Kj7lvDJ6zdumDOVE8Vryunz2YKHEDjYmT5aDJsMwk2iHdoikzCQ/jct9K7lqBU6LcrnxrLdJ0QMPSrwfr1CU1VSepSahZRcNyJt8CcFnnbXbi5u5NdeCpusnSti+4pikRVUMXIfWVRcG8zTX6+QyVbczOcWHL3e7S1LRU5zEm+VW7J5d8foXtNHdqV4+m0XcTS14KPyA3zgA8i5iR9yl5130xOgvVixnu5NrjJLRbSINxMUgsn0HK5zupElUcR6jUj3luoz9EzpSf7/gPSraBr9bjtd6P4S87Ifo5pOxNy4RIQ8M21Bd9PyOWKp8ybDhPwQ+Wb/4fFqRSG3qTWWBuELtpRAXV1VZ5r+y94VDGvEQo1gLnlcSYbLNY0E3SXWhZMBS7prhPqByXEGMWWlaEdCTHO0dcOdautd5+/iaHEJY4m7BrK8dGEjlluDhiCw/wih0xXfwsYt1ABZglWNxzUbc6X2lK1QFfUHUqlqVrgNYVW3j7TfSVVjcMIdg3G6e0Efo/c7zt9K6RCSyV39rP6r6Se42jNrsl+0hf5JVYmtpuuARzbo+LvlBxVh851pyTP2xmkia6ULKkPKKZ6i08Fwpwq02QXqXZR/zcX3vLio9MEAJKQAgpzjoQU3ytaUrBkDmU/zDEXpd9HPzQNxelxL5iLsNXhn9HO/FCNVtajc1hwCdUmAknx/Vra/z7wEoCSkgUUx4T7xp1g4AtAwCIpVwgmzDOqF+iqlSnDwQbdyqo0GJ+5cr5KWyPGlYy6ZJvci99mmgnhlTY1Q/r/GR0T/IRpe5K+Jtr7N6ziC59Oq0Czaz/uhoUteteWKZ1S9u1thpfF8hywQFhrSRj4S+1pBO1JOLC37Jr+0hlkCIMLT1CpYCbKCaKGfBtWlLHCsQZW3xLEgqWooUqjEmvo4qWhkYOfJi2Lwkox2Qvaj0trqCEH1T33HjyWxtc5wwQPkxPfRBZlNb6DCY4Nox0Tudz5fFo/bfKkyaSYJMZom6uK8z36UmHunJ+5LDDzg3hh3/VCXE48XV2vZ6IB9qPRcExc09zXAtWJ6FiDd8obKPaTbxrUFiw/dHB81BUiqajrTnZybokhAjV6v109Fl6/ld7ziq7G7XqaoDNVBRsOdkrtYvVrdsbkHda0f4ysaa8YT3/Hmy2/gdWaa6xoXhGK6sgRDbvb3Ez9LPCaJntErnpj/IfvY+cBtC/MpF+Akmt9VMuBGB5jv7p96DZYb5obatXCQJVhRTYu87eusWnKDM9qSIMWYXYjzTILrYj9VfP/40pTZOW5QAxy7ipBOMXK/gka4bWo+QLCevJrXdh5e/TBCb9q3OfpSb9YRBbLZnzvqvdg+bJRdY/Xa8tUpef29HW1EUBg2uM3T4A0cCXO3OquJ+O0p9RZcPMNrnVe5otzP4IbPfONG+rZlK7o1+L2PKxXOwf0Yw349+7ni/PufNdGTIy9B/2InEsDdFtYOCaysmDHdNhI3ep9yl72/aiuL9B26sJBP7ZwxvfM447PmoXRxfmtmmws/9wtmqxF7KXIW412gc5cfabvd8rdB4e1WUBQ9b/xwzfeHbesTFO5KU3zGFWCU+0oI92DspNoixXDSz6qAnRNGZhAJccTgkBToZP2R+kdaFdVdSsvrKSyGkZdX8jsOV+9uLgc6tDIt4x1HoWpuuwjBwreuRayjbQ4JNGFMOiKrQUGYTHBoqVUKZvXfjuSX5ZJL2vdTUJXR/hPi0jnLgOX5TLAOO9/+4iYILzKqRVnfpCt/fkCPXt9g4uS01/QpXOIOLAgvRdhvwhE5maPbYJzqn1awpgxfW1V7iPwukcpXseN+d4/DR+Yvj4QcjWKrddUpRthFybZ524swOMA2ulGUb2RPLfc42z1iUmjvdD7DJ6FcezdS+VnH5yO8bxpxnFxHi4juXN0nsiizGbOu4JT8blXMMbV+fd0tfzeoiMF1KeuYNyMzCsyZaV5tfSRssa6mDfSUiroPGDleo3fxJQ4rPIdVo+ToTfuqm+lK/YPkd3ERGvkZ1aIYvQOk7qfcli5tSJoVjtGiu9rBVUdlkLO1ow+1FpRrKPnBmuDTRVLcW78UZjxRzM77OJLeYNY/mL6/bIvazUHhhajT6PGx+4uWCzCV7d+xxJP3xsx+fl47t4xzxkTsooV4+zUkeh19DtlJWlMp8PII/tTZMCpOzP2WOKUcyv3kK4IoVqvKo5e2/URkTnVliXqZr9hy4KJnN5EJgBn2hyneT5QtsDCYIqpGoklVRDfLLBiHDJ4Ah48F38Xa4SBiN/b3wZ3JhLwoVy65kKPpBH71dGzJp+zpEqXvujWSZgRybyK0CbE1x2enk8UGTo31/g9Tp1Q4pSvJsnL+6rct+2HmAmNcmow4wEnw1JWpvO7ia1JPntuZu2xxU0eG+Ax/ZAaWpQ8WTbPKcrpCvsQkO98Wcfwfbam1Yq3VHG8h0IuI/3jip4FbqT9AKxu/2u6qqvAna9eG2YqaMyIghtrbYNxw6aHXteoUayOf4fg2JgmkFVEFoW9T2nY6MxBR6yT7FsquWW585/VXeQKqicToXJJjg803t9b9obxVmsk3by8sGpwU0LS0+PI+nr1tLL+73J5pN/p6O39H7n0AZjw7SpZusa555BQ7E7+6vICXYwUqi4aybrW+uqSwxhELOxqqmHXUQ3p+/jDfG51WLl3IiJbyjx1xdeo4m6odHhckMVlQj3axO+W4EIGM1Sed1zAvnTYJdA28RC2ZnkTyplw4hWxrcZRGXiElz+ektfsu6xSPlP1dO/LT657Th2IgmSNG0qqrhfBpX4taai8te7CdChxYwZHSNArnvcdIk11Jd5ixvE4kIEaVziC+soVVWpi0oK7Q8f4+uPF3byxUvgGUC4AO9qSTzfQbL2YkIisyJZVnu+j+2dYkUWtA+rArTQ9rtH5QS9VfIiKyYhdDgYldpmu5ihIYLqbvep6ruIqZ6aprGv7onmMQoPt2ooNJ0ra8MLhTbossdgU3M5mlZ99fo2e+VqJzxW3uvKScSjggDyw1zel1Pabz9H3Y0eDGEZhroXciZ4hpCmpoJnFtg99YtImwTO44IZpoWd1lft7X5r0lq4x2aNPk+YaZ0uFH6Mo3y/cIzETqMBMrBQu6MF0jBIrmNqbvk9CT7m8hGXRe5m75Oi2LWAn6yyAFLpF+4JUAUuIVBZSv2/ce7pDv1YCTMl3MqccPWNiu/juBDFJTtDS/ovaf2GB+V4zvfguHF80pMxWHI8m58fWofoa/tklgkXB1wVycl8Pv5Krg40ajEyKqfvr0uNZt0HQVFlGDiK0LeLK3QFmn9/9jhVFH10C8HfffX73++mH199953Jut1hhNsmTO6muY5Ys33rBfq8X7EbYJp1gWMRWInzNTtwuJc1zgIl9LvYJTJiVVFRoRmIKkI4rKQHGRXwvSCA+EAtotsNsPJz4wd4B6H0eG6i9PrFL1HW1THQpzDLXRsWufId67WQOse5bGu0drWs+0jlJjy12aQeDjVQaX2zS1r34ehcLYsUmHU31VpM5Yo/darAbUWCbw/KesFA+up/g/R0XFnmv/38Yr9qqzG7y36OwWN7x0XtEDiL5KMxRx3EP4SflDElbvZPt2KXPTJPRXmfZQZ/M5+B2G3Hu7ZHpumU1myMeBkVfK8y4pXXdzOXSy4yL825tG3TisuagoetAC4PprMI65zqzKuIR+zkm8RrSrX310ZksikoMPVEj7MRxjZseit17emP+g4Z16gY3fZxm/VDcrrDI/12Go2YtbgYbdoxkeDB244V7yOlKl4wwGS1LdC4LHrDfYSXGQYenjroWRZnJVML46v27S/Sb86O2SalhRL7Mmkpw9Z9v0ZeKqonerRUXmaLDTp1pkxs6DtE9+lAXnQXTuhotnUR8SLtAZewxAhZoeZTj6DaoJhAcezDcPP6ABsyxKhKclgWbwL2Ay4gFyA3QKo82lbYHM263qx7oHJuhVvhQuEsqyKbAKlZZSQN3X+LR+OIHR58wGaVTRYGZbaLzAqGruAVUDeDVGlotJQArl39PALXE0SdhuI5T0dkLgu4Zi/3g+M5tBbWqZ3SkRYYJDEaJX35iYWsR0XjvAF6uy+1P4sZsor/vRGTEqCzXUfuud6BbyMdFnu4AeMtxdIkhMirWTEQsihyDTpEbLbJVpnfMkOjyQ2QrLncaF/FzV7qwhdmmg54g6kJExkRKccJESVWx3EdLeB/BLsl1GuBbzFPwCiuzUkkjs/ghKYC+/SkDj2N82DzZ3eRyneUpiG0Bx89/IyIr8E1mTCy3QR+w5WhOEzwKBROJkGYiHdIl1xlf8ix2WLQH+08JgUfvDN6BHbsXYhd27KreLuyfE8J+lRD2PyeE/b8Swv5zGthGlhwvaQqR0kCPb56JrKg4KN/LfYJ3sgZeXifQS4qKs3VRptG+rZaJ+Tp2EpKHzFIoJZp+IfF9IyLTLiExwQlqRdJYkxZwGmtS73VVJphFSkRTVp3EVDXSWNOD3iQQIUYaa5ilgg1mTRLglWA3AgupKUnAhNtXliqJHoXtK1maDcV5AreaLMqM8AQ+bAs4QZAE4Krl3sR3i1rIOgnkssoSxDSIYoYRzBMUEOkMr6kg+4hZV13YAvP9HzRfpsB7m0Eb0CSQXTuYNFi7xNok0JfrcvsqjQ9aZ0tm/pyk0RjRWdxZcQPASkYX1TrJNQeolKj4VW7a+fijzdrqAKZm4/z88Z0jDjiofUmAu27y8TrIdWCvGKcpbBidrVIcIlvFLM7uA06hG+iMlZCkmCURdazc/pRrU46a+UeCrRVJApuzFU1hxmhwNBc0Z9EKRvuwmUjDJYXMK041kSmo7YGzdQLZJEu9wybqzP8O9FAGeRTAiq6ZNgrH94S0sBNofIqWqUitktFaQydylUi+usx8x+IJoBtFcZFAkXSlQKnQTqdc7zaS6cxNmI0PfY8VTsLg+UQhbAzIWzffPjZcpg0W0ecc59osKxVrWGANlbpZQSmgVtFxja9H1zXJscHC5IZV/GHXx3YaOARzjfM89h1geeywat06KMFbxIqMKCmLJF2JLOAEZhorsjTJkb7jUQoyl9fR2zOVOn7LUlbqUrHIQDk2zFTRs884EzRei50Wqo46UaeBC8W38d1aXLqup9mKy+jPeQM8Qcq/tXmjSx0LNIHEsTZ0AlSj5yZwuU7CumKd5AKXUsUWYMWyWqe4ZgXTJIVYKHQShk0xB0JQA82VosONLsNdA+jYGX8Oaux0PLHbxbZAklSUSTcAOrolKuNrRlKxdRaYx/VguDtBVfw3q8zcUN7oYKNOpm7BuhGvSZgsQeGmn4kTWxh4sLGlQZk5R1J0dLHW9sOMbGLV+Y9A05uSRQ8ElFQVa4WFGfXcjQF5lwRw/KfXdSL79GkwBTQCYCXXGdZlxIEBXdAKx4aqKOYp9DtFCdDBdR1NBDw+kS3kuC1cO5ClyhNgHN+RqRP4hrXzDSfIB9A0diKAG3icwDjR9Et8Bgg1aI0GNYEppdk6geDVZWwvm1YkxT1QJI+uSGtFQl1xIwA28UZsdWFWOnpXzS0RsQslgtNiHwrUNemMvX2zNvHZygGNH9FrZnrGhrsvo3drrfJlkjz0SvEEb2GlqcpyFrvqPcnYijoylIIMhmiDi9je4G3GhDZ4lUAz2DJlUqjh21IkaN1kpKpETDdrqC1aoKPoaWUk+lAJNFq6yR5JOCzvM+YsR2eK5sygM6xy381QQ/v3MDpuclZCKk1NCAUwMEQfQX8DIjkKleo0+RBMpKPc66Lkck9HgwVvpd9KVtGaet+RxywNnc8I5p0puqY3qMDDRgttLFasq+EwkORIcqZhOEO9uj96aKCEdFWWUhk0bjyK0G6DDWIGlYqupljhAWm59xlCESK8tzoaFBATvrP7RF9ozkTqifwdVO1qXTw1MnJNzYaqRft9vZHV6EVDSNAtVc04IiNRiZWm6B01GCaCu7uKGxI8eyvX+sWlK3t9js79iK8TZDaBKUXQDPgD9aOPAW2B3lPzOzOC6vA5j5k6CfFWMLK7uUWwuNuspliRzYIJFsQPZu7O0F97ID5hFgYkQ7zguBIw63ddwRzXuol7uIH7oF/7gT2lb8fd7Klpwu3nF08Y+/Ygsog1TXfrvArLoo/0xsCtmHIXzDGNekIgtYPr3sOEasEnJl5C99yE48Chf66mBin6paLaHGjafXy28v175TuVAcbyuFWdxB56pJq807475RBODiOIjfX+Dh3a9S/Bncec/X/7fEO72MV5LRRg7TBvgNUQL4n3nixsH5cl1hS5dO0GGzS6Vc0p+V88Dr6iGQXfYC6Va18fJCNCWCNNKYw7w4fnVSksNCYzjPcddZh2SwtQe1umIZWCCWiHkC6pKphTN+ZCul3SDeZgW8bpmiJOt5QjrDVbC3dw7bz+MOtDS+ZHlN+w/gFOXz7KpGeLWSXYl4oOxyTi8OXr4Htcx8TjpqDUGg3L3YUkUggKuRVox8xmSlAgFKgMaTR2RY8qL7q3aWHJCfKkeaK4XDOCObIYTJg+gMXjYgdLTYxpfDzalZu9DqPXSWfbyUFWa+wHHnOGdbaRyW0CZ8Q15hrMUmmHGlmp2B3BE+4HgNylsdjCm+YHsRBOsVqcci2tId67b+cQLEe/+l8s0KnYN/83gm7AltfCIJwviCzKylAVFsNJ3Ph2Y+nMs2+GZwEzFnsHwszfqpd/+uHP1vY97xxHTbFvgmh7Ps3iRszu6rjBe6rQPzc+Of3CowHIhW997Pqf9DwvWpx7XH/wPI5MXr5Ntn07HJhi11mg9799fG33ThV1zhPwl+ZME0VLLMjeapVePePDXBAEFDpBH9/9gi6E+fHlCbp4f/76v35Bny6EefUTerbb7JGgzGyoQmQjtR+VJpWixMC3fnj1v//H82+DFKFmk1DGDekBMnVR4PA4Hp2Y++55za8cL17USIWveP60kO7KplswP7Jh3J0f+BC+A8W0tU4+M2UqzNHb0/dBZP+QgqbzZR3HGf9XCroI09ai+9WIUNjI7cITjuApvsEHzmGNDd3hRxiRDtx9iU7zXIGf1nF5CJ3m6SVFeWyc86GxkIuzd5fuVZoMjxVYzxj96DmVnKbq3250cWlRmfB+WRoeOQkiCg3t2tM0rDWxzE3XmldAdNDFec7slzFvA7adWf7hd25GBrAmIVxw6W/4eZ8FRqi0udZJ9Lq7PmkYvfcYXkplGpE8Ero5BNjgAJjZ3y559cy0d/thYl0/JvW23k0RXtCQ3TiXF9djB5Yv1loSZlVO5zca6TjIymWFxZouGtOJSLFi60rRHC33AJOKHLKGwnKmPLL1wKhodEJbDi66StDvgEfU/bslXNEdAIoW0tDMZ3bHzzOKT9pc6AxnLhU/AejSqDTAVwlYYpWgWpinuA6p+p+UCYiK86z2xKVTy4cWvN3HYrha15nwCBrsa7OhSlCDPu5LeoI+1c/YW3CA/YguawfY6CX4bUpTq0f1zKBMTJjGNdLeL36CMOdBZaJsvwgJblhBYt6WKvsGMmEk0gYecybQp4tJgUIgQTaZvIousi1QWSYY+2YBK6pjZ/RasAlKXNyLGDsVHfztCbB1oxUyTsU6+qRIwNkqHwm10AkN1Kk8mHcCMAIRSCdYIYzeSLXDKh/P6UbodA3JXgphe+NvIJduSc2OUhFWPSN3TbxvjFsazLuhOocMgpbxkBkx2iETPs8V0hIKZqxY8iM2wlvccizmiOPfwUFZJ4h0XJSjDfZdlm0kZWst2DUYsP2XJ3akkhLoQrCN1w/ubhF7rAwjFccKQb9oVCPx7PXNL2/lWq5W4envlGRmQ5Mfbw/Zj3ZBdxs7eL+2eFt0TyuzocL4ZPFJtHUVs3PC3RJ63JLTqH/SVE0iLCtD5LyU9ktOI3xVEUK1nsAZOo8f1xztuMQTwAtZFXct1R4FChNGuM0hnHo40gGOVipBgE+XUth3xcqtkHLY/BCNFKX+rrbx+tFNvJsYua6lUDPAGc2b/Xg/zEAfZgJpZqqA/ERQXEC9iPZQN1gjnMvSvi5mQ5lCcifaI3OEM/hGCllM5NXCTA7NXIv6eZUIq9wzkVv5I5VuCIDRG8YpOvWILUZkuIuzVzQbc3dyMmG82f+jpCtMkuDKZy3EpUJojwFCxKx3fwAhXL7ela/XiE2J6YTQpUxZPRDY/JJu8JbJCrRLIotSyYJNZCjSuZF7LfCSQxHZCp0dxo2JbSN2EiI5xLCndaIgAj0Mow6XOQLBwPoNfqlPt/PKtvdtku3aMstKmGE5W2yNPocy8IwcY9bfSQuC93hNBVWM1FsCgkCi3zC1gJkNPLWh2W7II7sgPyy0UdPBz3pPx7TderQ9vTy8J69euLUS7itomjZGuGEF1VauO21P0ZJOBpH8KURrCnHrQUDjwQceg7ojax3Tu/vRWOvHu+3ph0xHG3J65615h/FtOxztDXbcCoQ7CIOvd3cvb92dmvXs3EWLsjd1+8lF66U6jwC5RY43AuTrZccfbz+yWKMN5jmyu8lHNasEiXnH7iA/ZmXHmHsbMWOj1EMJ2sBPHb1ypzKbrKBmIx8hSoJ7nmTk0PBfmzxw6KWkZFKv04GozgfJvb/WInKALxN5Qv5r8fOf/oSevT0/vXyOzpk2TKwrpjc0h1L4IC5crmXyvkCHImGQLbtyePhjhi9OZIwpmdireKj+055qCIPmxoBHPtrQ5/tcFwJp/03db8fxBziFYqZYhNqkt5limMfqTjfYyAecs0q7FZBUSLOCcayceLJi094hAu96uLwK7rlm+ZydRrqZ8p8sI9RexEFfzPaSp6uzOBWH7jqENXylYcf/651E8MmIF7zjhnbKMvKwK1OqlIkBo5ANkFqqNRbsjwNZ1SIdK9yV2EdQustTE+ReMRWsJU3U9eeNXQ5eC9fiy/Uu6mU1/0oxNxuCFUWlorksmMDBgruOeLrEhlFh9K3p8RzPudu3+FE361o/0jIR49qr860VXCVWBpohtVs9LFZnbHbkhc1dJOqK5lRhQ/MsWlLZAf6wwudNvWITPLtUcsvypnmY/x4uS+411RFj+OY/9lnr67RhBafdJMtn2mWzpO/1Z/YT2wwOD4XMyS1z0fPNUHGfaAHXKJ0xh4LfV/OkN6AzdX7UqYReBzbqdFTQWLFG2kjlJL6FVlCDYbVv4VsL+61vw7svWJ5zOp+Uewfr3VXOBY63I/eOknP1eIx5tnvpV+t0GBL7Ojp7gkqO7ZHZ91kqRAVR+3LKyw+pkDPYk3fIoFONbfmr1Aa9w2TDxIRJl+NEkuObIa0/Ccj0LxW14sPqR67JmV6gtzku0Wf4H6cf5VK4utO/jR9PtMFbajUnTrFCXyqq9gh6EOpSCk1rjSpcnGr3m8Fv5pGXvgcesZAVq7tACrd915dvGs96SzOg2jLQB98c9a6YwpSntA6zIY/XraV7TYysbegfXqaRqoQI2rH6pHl5XOTZtZGaqLHzEDNvYaY/CIx2TORyp5EuKWErRuwnJ6E6QZ8nO74gdnsO3zbnBj2DjrBUkPYZgtDl8w61UCXgHX9L15js0Sfdb3zbRGCLYSFt9Oxau8IMBvvEa981tQAVqFUDJrMv4ojiTR+AQPV/r9IUynnG5OtvO71CPdWd16nXgR3DDoOM5n9zxGbnyeud2qrP8PWu91rWvYatT3cBHe9mHoddEzDon02bkOmOYXRC4YYUtxc/Q9lAzJGAkxVusOWcrpjwvnoQTtDVr8DlRNNBwO6oQrFEuLUOmIH6F1swNj7b1Hv3vZQmelM2PmxjMNkUM7fAb1cFgqORddQ9jiRDXpZMxJsgFvVu2C1DUWHaxzMgpLplO3Asro12W94fmNo5wjrt23cL1iVWNU/ZP5+0W9lt2KiVOrK3w9qyLvn9Ttsz0WeWuLYWUu3THfhfdInFv93aMaZGpN9FvVbPQ0+TJctfXgD0W/b2aCrRaFd1v/XDu5rkgowKo2R5jOjIZbUcORfuxON+TWtt01vKEQBHV90x7z08k0WJxb65j3DtYJy+s1e2VNlnKGNiJcNKAdbXqWuEbpEfAyuyxmxH03ZFX31JlSPwpuJ8j/6zwpytGM3ROdQ9O+dgEJUdXWZEymv2SEH33+kSufVb+xnzKW0+erfZNhxeVgZU7iNHmN5+1z80S/gpO94d7XzyC/RxX7qtt54DSxx3gtOHp+gqi9pMdoC2xcE5ItS3OtS2dojMHK66RrnsY+c8i6VUtbcfQswf3k4ceadXTmR2qmlRpp1DdIAUduVbPfc1mkrKRJpIHym7jj0PVGITdk0SkWEdM9rfAax8OX1kyJXiEY+5AzXiqTTGaFapWN6QDkxNVYbX8WzKFnT056kPOmr6Yx+05/oEgoXeGCpAtYpvnFj40bi5UfQ2ig5SZWJrVG6JOWoJezL3IywL6tUL/99nHoUX/j98XlPI7Y85VeHsPL+dR4yeu810g+fgce2MWhttJ/cD0axJxcSKKjURdx3ve5Z9dRX/W0kfdM/OgGTdl3jVOYbAlYKwtkx6pQJLzMZ+r13c3rLdR8ggVt0//ZWOE7SmB36yckPVPP4Iq7P7jKdnZzD68Tk6g/XDqFFlZmqWMkHnM6r88E/ay8I80JyXJg0ddwjZOXC76Le60yn64EmzP471St6/NUr4tNEV+yPsrWHXiWTKxV9fI0HX0jB3gOUG64kJUJrM3Vaoc5Ru8enhgvaok02AGiW4DHisbpxe19+EE1I0W89RUdHvb9RMPfw4OWjZShOmdRVd6QTIkCyVzlv3sBgKYEiVSuoDHR1KV3q+toujKwhOH5JOs2RINJ3BfRT52RWkdh5+jDrS8zgk7y89D+A4LUK15tk25Ys+DKl6R3YQmTyzrIer6G0adSrA7Jp6izpRc4Nv2nEl3QcJZOtPSEO8Tip0cXX613eX6NK+U+g3MTF9pcU2USX1Mdh+3MkwtiCGyIaSa32UE/luQjhtD7LQ0LmmX2fTIgzSQP0IwlYKHtByqWKjppCPoOQ6PJquIJNGA+BssKlmm/DZxXKLOcsdIwaQGArC2bpaHxKEQLFrutdDsR2J8+sE0siwN8aUOmMwgzYJaDjKFAQh+AncJrYWdeWLVMzsb7lRRBZF0j5xd8Tb4eEdQuES/B1TlA8tzdgulh3HItP6sQbe2pWdDP/d77au0Qpi60qNs1KyOdKqQwg7DBBgAEiFrQEgK9lgIUaNM1K3m/KrAiITMduZ2jY3D4ufefj729P3/t17MVi+eVCMVEPff/SebUxfZ1vJq1QEOK3nOAs/56aZjF2P860EMxo9c0jo59CtAwp764m6A/AIkA7uhleJpNlbj+snwYxPF1j0iw62VEGmwKriiEhBaGmsoXzlznCivcJul1L6OsJbg70eoW0RLaUySFr6/vrvp6EU3CDZY/OdVOv5EyyHBQY9F+sSu2YnwUYx//H6t8uLS/QO3xRM5M1Y7/Cx2r3NnobZG6I4sS2/jdHuDm2rUZ/CJYvR07NdlWO2mq9g87GL8OstJ1c7es4yL5Uvzn2XXo/FQQz5fIfyyL0C6h0X/+3rhpvCHJGPNcnYtxv8JdaEfqTsRj+uGqz4JqhbuOLeE6SrQIo61ugv2igp1v+25Jhcc6YNzf/ywv/tpPmUiRUl4Y9WTNEd5kFFBi955zcIixxpiSbYUtE100btrWU/p7Aosdn4Zv0NDmiIwwhJcErNhaYrhHb1WkSqThfyRp9sMKfCdHJSWo+7JnJRFUtFOe9a82FO7+HTT79/A1cAbuyZBYo+eaDdh3V8Twbt5tjAYpkmzgFUEDoVCCuFm/z7nK2gjNV01kGKcoj1+DOGutaQFuAdjpFQ+wjZK6RyrgrlquvakRGsrmUf6m0FNmRDdVB5lZyRfVZHDMeBwQegCs2BmmCk60zhUGlaDDJddyBZoNMtZhziZG32PfoRbjheym1Qy+rhHY3Gxnd9a1G3VC1w7psd1Bi/kQrRG1yUnJ6gDxIXTKyhrqAy0BaqnqgawnzJJbmmeRafQ4bcoKC8vi3C7nLGklqUPS4TR/DT4SPwTBiVc+oD2EF2PcA/sX/0CeaG3pgXG1PwED56gzO9wS9/fhUDm1/pDcrZmmpTy4N+14fwtcfbLKfGTf+Ndq4NRO8aIESqzlQYhIVhW6YqjahYM9EOWIHKFiZ06X4eFAMVjiM8kX3vwSnHOSqlJRBzRQFih4Xlwk43IvTs8tPpc8+gugnXlEreMKvBWbyxlRCmUqJT1tdsVBMsRH98b3MERZnlTJdSs5HW+hDxC/GMbtWhbvAFXQQwAlTdU3aabzG0QHiH+c4q35dK1uf47PTd5XO7wxKrhr/qt88Nhblojg2tKGRQ/Asi2F5cdMYpFicWLiNMVvCWfxLXQu6CR2wJUjgcxv67IylysWqXPxmNUvOr9Tn19N3lFHaaSBVNhACwISaQA2oxcAoRqBRNbbrTdev6FXuYO8a5pfSSYxEU4jk2mNDRWIAHoN2lX8MJ59hgdAbrOJHuiwF9HWilqfoe6vWdTqLwasVICF83wXBoOT8AXW8VNw9lE4p2s7pNJQTli3/6/wIAAP//a6iF4Q==" } diff --git a/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml b/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml new file mode 100644 index 00000000000..e45d12ef732 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/_meta/fields.yml @@ -0,0 +1,61 @@ +- name: cisco.umbrella + type: group + description: > + Fields for Cisco Umbrella. + fields: + - name: identities + type: keyword + description: > + An array of the different identities related to the event. + - name: categories + type: keyword + description: > + The security or content categories that the destination matches. + - name: policy_identity_type + type: keyword + description: > + The first identity type matched with this request. Available in version 3 and above. + - name: identity_types + type: keyword + description: > + The type of identity that made the request. For example, Roaming Computer or Network. + - name: blocked_categories + type: keyword + description: > + The categories that resulted in the destination being blocked. Available in version 4 and above. + - name: content_type + type: keyword + description: > + The type of web content, typically text/html. + - name: sha_sha256 + type: keyword + description: > + Hex digest of the response content. + - name: av_detections + type: keyword + description: > + The detection name according to the antivirus engine used in file inspection. + - name: puas + type: keyword + description: > + A list of all potentially unwanted application (PUA) results for the proxied file as returned by the antivirus scanner. + - name: amp_disposition + type: keyword + description: > + The status of the files proxied and scanned by Cisco Advanced Malware Protection (AMP) as part of the Umbrella File Inspection feature; can be Clean, Malicious or Unknown. + - name: amp_malware_name + type: keyword + description: > + If Malicious, the name of the malware according to AMP. + - name: amp_score + type: keyword + description: > + The score of the malware from AMP. This field is not currently used and will be blank. + - name: datacenter + type: keyword + description: > + The name of the Umbrella Data Center that processed the user-generated traffic. + - name: origin_id + type: keyword + description: > + The unique identity of the network tunnel. diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml new file mode 100644 index 00000000000..8b0ccde6e2e --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/config/input.yml @@ -0,0 +1,23 @@ +{{ if eq .input "s3" }} + +type: s3 +queue_url: {{ .queue_url }} +access_key_id: {{ .access_key_id }} +secret_access_key: {{ .secret_access_key }} + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +processors: +- add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml new file mode 100644 index 00000000000..2a602ff2331 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/ingest/pipeline.yml @@ -0,0 +1,246 @@ +description: Pipeline for parsing cisco umbrella logs +processors: +- set: + field: observer.vendor + value: Cisco +- set: + field: observer.product + value: Umbrella +- set: + field: event.ingested + value: "{{_ingest.timestamp}}" +- set: + field: event.original + value: "{{message}}" +############ +# DNS Logs # +############ +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - source.user.name + - cisco.umbrella.identities + - source.address + - destination.address + - cisco.umbrella.action + - dns.question.type + - dns.response_code + - destination.domain + - cisco.umbrella.categories + - cisco.umbrella.policy_identity_type + - cisco.umbrella.identity_types + - cisco.umbrella.blocked_categories + if: ctx?.log?.file?.path.contains('dnslogs') + +- set: + field: observer.type + value: dns + if: ctx?.log?.file?.path.contains('dnslogs') +########### +# IP Logs # +########### +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - source.user.name + - source.address + - source.port + - destination.address + - destination.port + - cisco.umbrella.categories + if: ctx?.log?.file?.path.contains('iplogs') + +- set: + field: observer.type + value: firewall + if: ctx?.log?.file?.path.contains('iplogs') + +############## +# Proxy Logs # +############## +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - cisco.umbrella.identities + - source.address + - source.nat.ip + - destination.address + - cisco.umbrella.content_type + - cisco.umbrella.verdict + - url.full + - http.request.referrer + - user_agent.original + - http.response.status_code + - http.request.bytes + - http.response.bytes + - http.response.body.bytes + - cisco.umbrella.sha_sha256 + - cisco.umbrella.categories + - cisco.umbrella.av_detections + - cisco.umbrella.puas + - cisco.umbrella.amp_disposition + - cisco.umbrella.amp_malware_name + - cisco.umbrella.amp_score + - cisco.umbrella.identity_types + - cisco.umbrella.blocked_categories + if: ctx?.log?.file?.path.contains('proxylogs') + +- set: + field: observer.type + value: proxy + if: ctx?.log?.file?.path.contains('proxylogs') + +####################### +# Cloud Firewall Logs # +####################### +- csv: + field: message + target_fields: + - cisco.umbrella._tmp_time + - cisco.umbrella.origin_id + - source.user.name + - cisco.umbrella.identity_types + - cisco.umbrella.direction + - network.transport + - source.bytes + - source.address + - source.port + - destination.address + - destination.port + - cisco.umbrella.datacenter + - cisco.umbrella.ruleid + - cisco.umbrella.verdict + if: ctx?.log?.file?.path.contains('cloudfirewalllogs') + +- set: + field: observer.type + value: firewall + if: ctx?.log?.file?.path.contains('cloudfirewalllogs') + +# Identifies is a field that includes any sort of username, device or other asset that is included in the request. +# Converting this to an array to make it easier to use in searches and visualizations +- split: + field: cisco.umbrella.identities + separator: "," + preserve_trailing: false + if: "ctx?.log?.file?.path.contains('dnslogs') && ctx?.cisco?.umbrella?.identities != null" + +###################### +# General ECS Fields # +###################### +# This field is always in UTC, so no timezone should need to be set +- date: + field: cisco.umbrella._tmp_time + target_field: "@timestamp" + formats: + - "yyyy-MM-dd HH:mm:ss" + +################## +# DNS ECS Fields # +################## +- set: + field: dns.type + value: query + if: ctx?.cisco?.umbrella?.action != null + +###################### +# Network ECS Fields # +###################### +- lowercase: + field: cisco.umbrella.direction + target_field: network.direction + if: ctx?.cisco?.umbrella?.direction != null + +################### +# Rule ECS Fields # +################### +- rename: + field: cisco.umbrella.ruleid + target_field: rule.id + if: ctx?.cisco?.umbrella?.ruleid != null + +#################### +# Event ECS Fields # +#################### +- set: + field: event.action + value: "dns-request-{{cisco.umbrella.action}}" + if: ctx?.cisco?.umbrella?.action != null +- set: + field: event.category + value: network + if: ctx?.cisco?.umbrella?.action != null +- append: + field: event.type + value: allowed + if: "ctx?.cisco?.umbrella?.action == 'Allowed' || ctx?.cisco?.umbrella?.verdict == 'ALLOWED' || ctx?.cisco?.umbrella?.verdict == 'ALLOW'" +- append: + field: event.type + value: denied + if: "ctx?.cisco?.umbrella?.action == 'Blocked' || ctx?.cisco?.umbrella?.verdict == 'BLOCKED' || ctx?.cisco?.umbrella?.verdict == 'BLOCK'" +- append: + field: event.type + value: connection + if: ctx?.cisco?.umbrella?.action != null + +# Converting address fields to either ip or domain +- grok: + field: source.address + patterns: + - "(?:%{IP:source.ip}|%{GREEDYDATA:source.domain})" + ignore_failure: true +- grok: + field: destination.address + patterns: + - "(?:%{IP:destination.ip}|%{GREEDYDATA:destination.domain})" + ignore_failure: true + +###################### +# Related ECS Fields # +###################### +- append: + field: related.user + value: "{{source.user.name}}" + if: ctx?.source?.user?.name != null +- append: + field: related.ip + value: "{{source.ip}}" + if: ctx?.source?.ip != null +- append: + field: related.ip + value: "{{source.nat.ip}}" + if: ctx?.source?.nat?.ip != null +- append: + field: related.ip + value: "{{destination.ip}}" + if: ctx?.destination?.ip != null +- append: + field: related.hosts + value: "{{source.domain}}" + if: ctx?.source?.domain != null +- append: + field: related.hosts + value: "{{destination.domain}}" + if: ctx?.destination?.domain != null +- append: + field: related.hash + value: "{{cisco.umbrella.sha_sha256}}" + if: ctx?.cisco?.umbrella?.sha_sha256 != null + +########### +# Cleanup # +########### +- remove: + field: + - cisco.umbrella._tmp_time + - cisco.umbrella.direction + - cisco.umbrella.action + - cisco.umbrella.verdict + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/cisco/umbrella/manifest.yml b/x-pack/filebeat/module/cisco/umbrella/manifest.yml new file mode 100644 index 00000000000..3a7150e714d --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/manifest.yml @@ -0,0 +1,8 @@ +module_version: "1.0" + +var: + - name: tags + default: [cisco-umbrella, forwarded] + +ingest_pipeline: ingest/pipeline.yml +input: config/input.yml diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log new file mode 100644 index 00000000000..3e5f23fced2 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log @@ -0,0 +1,2 @@ +2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW +2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json new file mode 100644 index 00000000000..65aabab5a88 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-cloudfirewalllogs.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-07-23T18:03:46.000Z", + "cisco.umbrella.datacenter": "ams1.edc", + "cisco.umbrella.identity_types": "CDFW Tunnel Device", + "cisco.umbrella.origin_id": "[211039844]", + "destination.address": "146.112.255.129", + "destination.ip": "146.112.255.129", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,OUTBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,ALLOW", + "network.direction": "outbound", + "network.transport": "1", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "172.17.3.4", + "146.112.255.129" + ], + "related.user": [ + "Passive Monitor" + ], + "rule.id": "12", + "service.type": "cisco", + "source.address": "172.17.3.4", + "source.bytes": "84", + "source.ip": "172.17.3.4", + "source.user.name": "Passive Monitor" + }, + { + "@timestamp": "2020-07-23T18:03:46.000Z", + "cisco.umbrella.datacenter": "ams1.edc", + "cisco.umbrella.identity_types": "CDFW Tunnel Device", + "cisco.umbrella.origin_id": "[211039844]", + "destination.address": "146.112.255.129", + "destination.ip": "146.112.255.129", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "event.type": [ + "denied" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 128, + "message": "2020-07-23 18:03:46,[211039844],Passive Monitor,CDFW Tunnel Device,INBOUND,1,84,172.17.3.4,,146.112.255.129,,ams1.edc,12,BLOCK", + "network.direction": "inbound", + "network.transport": "1", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "172.17.3.4", + "146.112.255.129" + ], + "related.user": [ + "Passive Monitor" + ], + "rule.id": "12", + "service.type": "cisco", + "source.address": "172.17.3.4", + "source.bytes": "84", + "source.ip": "172.17.3.4", + "source.user.name": "Passive Monitor" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log new file mode 100644 index 00000000000..403c1c9df33 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log @@ -0,0 +1,2 @@ +"2020-07-23 23:49:54","elasticuser","elasticuser2","some other identity","192.168.1.1","8.8.8.8","Allowed","1 (A)","NOERROR","elastic.co.","Software/Technology,Business Services,Application","Test Policy Name","SomeIdentityType","" +"2020-07-23 23:50:25","elasticuser","elasticuser2","some other identity","192.168.1.1","4.4.4.4","Blocked","1 (A)","NOERROR","elastic.co/something.","Chat,Instant Messaging,Block List,Application","Test Policy Name","SomeIdentityType","BlockedCategories" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json new file mode 100644 index 00000000000..81b1478da27 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-dnslogs.log-expected.json @@ -0,0 +1,92 @@ +[ + { + "@timestamp": "2020-07-23T23:49:54.000Z", + "cisco.umbrella.blocked_categories": "SomeIdentityType", + "cisco.umbrella.categories": "elastic.co.", + "cisco.umbrella.identities": [ + "elasticuser2" + ], + "cisco.umbrella.identity_types": "Test Policy Name", + "cisco.umbrella.policy_identity_type": "Software/Technology,Business Services,Application", + "destination.address": "192.168.1.1", + "destination.domain": "NOERROR", + "destination.ip": "192.168.1.1", + "dns.question.type": "Allowed", + "dns.response_code": "1 (A)", + "dns.type": "query", + "event.action": "dns-request-8.8.8.8", + "event.category": "network", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:49:54\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"8.8.8.8\\\",\\\"Allowed\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co.\\\",\\\"Software/Technology,Business Services,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"\\\"", + "event.type": [ + "connection" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-07-23 23:49:54\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"8.8.8.8\",\"Allowed\",\"1 (A)\",\"NOERROR\",\"elastic.co.\",\"Software/Technology,Business Services,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"\"", + "observer.product": "Umbrella", + "observer.type": "dns", + "observer.vendor": "Cisco", + "related.hosts": [ + "some other identity", + "NOERROR" + ], + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "some other identity", + "source.domain": "some other identity", + "source.user.name": "elasticuser" + }, + { + "@timestamp": "2020-07-23T23:50:25.000Z", + "cisco.umbrella.blocked_categories": "SomeIdentityType", + "cisco.umbrella.categories": "elastic.co/something.", + "cisco.umbrella.identities": [ + "elasticuser2" + ], + "cisco.umbrella.identity_types": "Test Policy Name", + "cisco.umbrella.policy_identity_type": "Chat,Instant Messaging,Block List,Application", + "destination.address": "192.168.1.1", + "destination.domain": "NOERROR", + "destination.ip": "192.168.1.1", + "dns.question.type": "Blocked", + "dns.response_code": "1 (A)", + "dns.type": "query", + "event.action": "dns-request-4.4.4.4", + "event.category": "network", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:50:25\\\",\\\"elasticuser\\\",\\\"elasticuser2\\\",\\\"some other identity\\\",\\\"192.168.1.1\\\",\\\"4.4.4.4\\\",\\\"Blocked\\\",\\\"1 (A)\\\",\\\"NOERROR\\\",\\\"elastic.co/something.\\\",\\\"Chat,Instant Messaging,Block List,Application\\\",\\\"Test Policy Name\\\",\\\"SomeIdentityType\\\",\\\"BlockedCategories\\\"", + "event.type": [ + "connection" + ], + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 232, + "message": "\"2020-07-23 23:50:25\",\"elasticuser\",\"elasticuser2\",\"some other identity\",\"192.168.1.1\",\"4.4.4.4\",\"Blocked\",\"1 (A)\",\"NOERROR\",\"elastic.co/something.\",\"Chat,Instant Messaging,Block List,Application\",\"Test Policy Name\",\"SomeIdentityType\",\"BlockedCategories\"", + "observer.product": "Umbrella", + "observer.type": "dns", + "observer.vendor": "Cisco", + "related.hosts": [ + "some other identity", + "NOERROR" + ], + "related.ip": [ + "192.168.1.1" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "some other identity", + "source.domain": "some other identity", + "source.user.name": "elasticuser" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log new file mode 100644 index 00000000000..6200aeab3ae --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log @@ -0,0 +1,2 @@ +"2020-08-26 20:32:46","elasticuser","192.168.1.1","0","8.8.8.8","0","Test Category" +"2020-08-26 20:32:45","elasticuser","192.168.1.1","61095","8.8.8.8","445","Test Category" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json new file mode 100644 index 00000000000..4d25464cb61 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-iplogs.log-expected.json @@ -0,0 +1,60 @@ +[ + { + "@timestamp": "2020-08-26T20:32:46.000Z", + "cisco.umbrella.categories": "Test Category", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "destination.port": "0", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-08-26 20:32:46\\\",\\\"elasticuser\\\",\\\"192.168.1.1\\\",\\\"0\\\",\\\"8.8.8.8\\\",\\\"0\\\",\\\"Test Category\\\"", + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-08-26 20:32:46\",\"elasticuser\",\"192.168.1.1\",\"0\",\"8.8.8.8\",\"0\",\"Test Category\"", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "8.8.8.8" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.port": "0", + "source.user.name": "elasticuser" + }, + { + "@timestamp": "2020-08-26T20:32:45.000Z", + "cisco.umbrella.categories": "Test Category", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "destination.port": "445", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-08-26 20:32:45\\\",\\\"elasticuser\\\",\\\"192.168.1.1\\\",\\\"61095\\\",\\\"8.8.8.8\\\",\\\"445\\\",\\\"Test Category\\\"", + "fileset.name": "umbrella", + "input.type": "log", + "log.offset": 84, + "message": "\"2020-08-26 20:32:45\",\"elasticuser\",\"192.168.1.1\",\"61095\",\"8.8.8.8\",\"445\",\"Test Category\"", + "observer.product": "Umbrella", + "observer.type": "firewall", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "8.8.8.8" + ], + "related.user": [ + "elasticuser" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.port": "61095", + "source.user.name": "elasticuser" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log new file mode 100644 index 00000000000..bfe70c6839a --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log @@ -0,0 +1,3 @@ +"2020-07-23 23:48:56","elasticuser, someotheruser","192.168.1.1","1.1.1.1","8.8.8.8","","ALLOWED","https://elastic.co/blog/ext_id=Anyclip","https://google.com/elastic","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36","200","850","","","","Business Services","AVDetectionName","Malicious","MalwareName","","","Roaming Computers","" +"2020-07-23 23:48:56","elasticuser, someotheruser","192.168.1.1","1.1.1.1","8.8.8.8","","BLOCKED","https://elastic.co/blog/ext_id=Anyclip","https://google.com/elastic","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36","200","850","","","","Business Services","AVDetectionName","Malicious","MalwareName","","","Roaming Computers","" +"2017-10-02 23:52:53","elasticuser","ActiveDirectoryUserName,ADSite,Network","192.192.192.135","1.1.1.91","","ALLOWED","http://google.com/the.js","www.google.com","Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36","200","562","1489","","","","","","","","Networks" diff --git a/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json new file mode 100644 index 00000000000..fd474d2d029 --- /dev/null +++ b/x-pack/filebeat/module/cisco/umbrella/test/umbrella-proxylogs.log-expected.json @@ -0,0 +1,115 @@ +[ + { + "@timestamp": "2020-07-23T23:48:56.000Z", + "cisco.umbrella.amp_disposition": "MalwareName", + "cisco.umbrella.av_detections": "AVDetectionName", + "cisco.umbrella.categories": "Business Services", + "cisco.umbrella.identities": "elasticuser, someotheruser", + "cisco.umbrella.identity_types": "Roaming Computers", + "cisco.umbrella.puas": "Malicious", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:48:56\\\",\\\"elasticuser, someotheruser\\\",\\\"192.168.1.1\\\",\\\"1.1.1.1\\\",\\\"8.8.8.8\\\",\\\"\\\",\\\"ALLOWED\\\",\\\"https://elastic.co/blog/ext_id=Anyclip\\\",\\\"https://google.com/elastic\\\",\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\\\",\\\"200\\\",\\\"850\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Business Services\\\",\\\"AVDetectionName\\\",\\\"Malicious\\\",\\\"MalwareName\\\",\\\"\\\",\\\"\\\",\\\"Roaming Computers\\\",\\\"\\\"", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "http.request.bytes": "850", + "http.request.referrer": "https://google.com/elastic", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 0, + "message": "\"2020-07-23 23:48:56\",\"elasticuser, someotheruser\",\"192.168.1.1\",\"1.1.1.1\",\"8.8.8.8\",\"\",\"ALLOWED\",\"https://elastic.co/blog/ext_id=Anyclip\",\"https://google.com/elastic\",\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\",\"200\",\"850\",\"\",\"\",\"\",\"Business Services\",\"AVDetectionName\",\"Malicious\",\"MalwareName\",\"\",\"\",\"Roaming Computers\",\"\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "1.1.1.1", + "8.8.8.8" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "1.1.1.1", + "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" + }, + { + "@timestamp": "2020-07-23T23:48:56.000Z", + "cisco.umbrella.amp_disposition": "MalwareName", + "cisco.umbrella.av_detections": "AVDetectionName", + "cisco.umbrella.categories": "Business Services", + "cisco.umbrella.identities": "elasticuser, someotheruser", + "cisco.umbrella.identity_types": "Roaming Computers", + "cisco.umbrella.puas": "Malicious", + "destination.address": "8.8.8.8", + "destination.ip": "8.8.8.8", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2020-07-23 23:48:56\\\",\\\"elasticuser, someotheruser\\\",\\\"192.168.1.1\\\",\\\"1.1.1.1\\\",\\\"8.8.8.8\\\",\\\"\\\",\\\"BLOCKED\\\",\\\"https://elastic.co/blog/ext_id=Anyclip\\\",\\\"https://google.com/elastic\\\",\\\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\\\",\\\"200\\\",\\\"850\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Business Services\\\",\\\"AVDetectionName\\\",\\\"Malicious\\\",\\\"MalwareName\\\",\\\"\\\",\\\"\\\",\\\"Roaming Computers\\\",\\\"\\\"", + "event.type": [ + "denied" + ], + "fileset.name": "umbrella", + "http.request.bytes": "850", + "http.request.referrer": "https://google.com/elastic", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 399, + "message": "\"2020-07-23 23:48:56\",\"elasticuser, someotheruser\",\"192.168.1.1\",\"1.1.1.1\",\"8.8.8.8\",\"\",\"BLOCKED\",\"https://elastic.co/blog/ext_id=Anyclip\",\"https://google.com/elastic\",\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36\",\"200\",\"850\",\"\",\"\",\"\",\"Business Services\",\"AVDetectionName\",\"Malicious\",\"MalwareName\",\"\",\"\",\"Roaming Computers\",\"\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.ip": [ + "192.168.1.1", + "1.1.1.1", + "8.8.8.8" + ], + "service.type": "cisco", + "source.address": "192.168.1.1", + "source.ip": "192.168.1.1", + "source.nat.ip": "1.1.1.1", + "url.full": "https://elastic.co/blog/ext_id=Anyclip", + "user_agent.original": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.89 Safari/537.36" + }, + { + "@timestamp": "2017-10-02T23:52:53.000Z", + "cisco.umbrella.amp_score": "Networks", + "cisco.umbrella.identities": "elasticuser", + "destination.address": "1.1.1.91", + "destination.ip": "1.1.1.91", + "event.dataset": "cisco.umbrella", + "event.module": "cisco", + "event.original": "\\\"2017-10-02 23:52:53\\\",\\\"elasticuser\\\",\\\"ActiveDirectoryUserName,ADSite,Network\\\",\\\"192.192.192.135\\\",\\\"1.1.1.91\\\",\\\"\\\",\\\"ALLOWED\\\",\\\"http://google.com/the.js\\\",\\\"www.google.com\\\",\\\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\\\",\\\"200\\\",\\\"562\\\",\\\"1489\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"\\\",\\\"Networks\\\"", + "event.type": [ + "allowed" + ], + "fileset.name": "umbrella", + "http.request.bytes": "562", + "http.request.referrer": "www.google.com", + "http.response.bytes": "1489", + "http.response.status_code": "200", + "input.type": "log", + "log.offset": 798, + "message": "\"2017-10-02 23:52:53\",\"elasticuser\",\"ActiveDirectoryUserName,ADSite,Network\",\"192.192.192.135\",\"1.1.1.91\",\"\",\"ALLOWED\",\"http://google.com/the.js\",\"www.google.com\",\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\",\"200\",\"562\",\"1489\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"Networks\"", + "observer.product": "Umbrella", + "observer.type": "proxy", + "observer.vendor": "Cisco", + "related.hosts": [ + "ActiveDirectoryUserName,ADSite,Network" + ], + "related.ip": [ + "192.192.192.135", + "1.1.1.91" + ], + "service.type": "cisco", + "source.address": "ActiveDirectoryUserName,ADSite,Network", + "source.domain": "ActiveDirectoryUserName,ADSite,Network", + "source.nat.ip": "192.192.192.135", + "url.full": "http://google.com/the.js", + "user_agent.original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/coredns/log/config/coredns.yml b/x-pack/filebeat/module/coredns/log/config/coredns.yml index be7f27f551f..4f9992ae2d4 100644 --- a/x-pack/filebeat/module/coredns/log/config/coredns.yml +++ b/x-pack/filebeat/module/coredns/log/config/coredns.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml index 689bd725530..2af0eeca092 100644 --- a/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml +++ b/x-pack/filebeat/module/crowdstrike/falcon/config/falcon.yml @@ -23,4 +23,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/cyberark/corepas/ingest/pipeline.yml b/x-pack/filebeat/module/cyberark/corepas/ingest/pipeline.yml index ffe90e79f85..4e401931415 100644 --- a/x-pack/filebeat/module/cyberark/corepas/ingest/pipeline.yml +++ b/x-pack/filebeat/module/cyberark/corepas/ingest/pipeline.yml @@ -55,7 +55,7 @@ processors: ignore_missing: true - append: field: related.hosts - value: '{{host.hostname server.domain}}' + value: '{{host.hostname}}' allow_duplicates: false if: ctx?.host?.hostname != null && ctx.host?.hostname != '' - append: diff --git a/x-pack/filebeat/module/cyberark/corepas/test/generated.log-expected.json b/x-pack/filebeat/module/cyberark/corepas/test/generated.log-expected.json index 4056ed473ca..90805d72bcd 100644 --- a/x-pack/filebeat/module/cyberark/corepas/test/generated.log-expected.json +++ b/x-pack/filebeat/module/cyberark/corepas/test/generated.log-expected.json @@ -20,9 +20,9 @@ "10.208.15.216" ], "related.user": [ + "quasiarc", "itv", - "utl", - "quasiarc" + "utl" ], "rsa.db.index": "nes", "rsa.internal.event_desc": "pexe", @@ -67,13 +67,13 @@ "iatnu3810.mail.localdomain" ], "related.ip": [ - "10.92.136.230", - "10.175.75.18" + "10.175.75.18", + "10.92.136.230" ], "related.user": [ - "dolore", + "orev", "nnumqu", - "orev" + "dolore" ], "rsa.db.database": "umdo", "rsa.db.index": "vol", @@ -130,13 +130,13 @@ "anti4454.api.example" ], "related.ip": [ - "10.46.185.46", - "10.51.132.10" + "10.51.132.10", + "10.46.185.46" ], "related.user": [ - "incid", + "serror", "nse", - "serror" + "incid" ], "rsa.db.database": "byC", "rsa.db.index": "tur", @@ -197,9 +197,9 @@ "10.53.192.140" ], "related.user": [ + "psumquia", "atcup", - "ptass", - "psumquia" + "ptass" ], "rsa.db.database": "aperi", "rsa.db.index": "llumd", @@ -253,9 +253,9 @@ "10.81.199.122" ], "related.user": [ + "oremips", "eos", - "giatq", - "oremips" + "giatq" ], "rsa.db.index": "tempo", "rsa.internal.event_desc": "uian", @@ -304,9 +304,9 @@ "10.139.186.201" ], "related.user": [ - "uam", "tcupida", - "aboris" + "aboris", + "uam" ], "rsa.db.database": "isiu", "rsa.db.index": "iatisu", @@ -367,9 +367,9 @@ "10.104.111.129" ], "related.user": [ + "ele", "etconsec", - "ipis", - "ele" + "ipis" ], "rsa.db.database": "riat", "rsa.db.index": "umdolor", @@ -423,9 +423,9 @@ "10.116.120.216" ], "related.user": [ - "quiratio", + "animi", "umdo", - "animi" + "quiratio" ], "rsa.db.index": "oll", "rsa.internal.event_desc": "rumet", @@ -470,13 +470,13 @@ "isqu7224.localdomain" ], "related.ip": [ - "10.57.40.29", - "10.62.54.220" + "10.62.54.220", + "10.57.40.29" ], "related.user": [ "psum", - "rnatura", - "taevi" + "taevi", + "rnatura" ], "rsa.db.database": "emeumfug", "rsa.db.index": "omn", @@ -530,9 +530,9 @@ "10.74.237.180" ], "related.user": [ - "ema", + "tnon", "cup", - "tnon" + "ema" ], "rsa.db.index": "remeumf", "rsa.internal.event_desc": "lup", @@ -618,8 +618,8 @@ "10.74.253.127" ], "related.user": [ - "onproide", "icab", + "onproide", "tema" ], "rsa.db.index": "mqui", @@ -664,8 +664,8 @@ "tlabo6088.www.localdomain" ], "related.ip": [ - "10.189.109.245", - "10.92.8.15" + "10.92.8.15", + "10.189.109.245" ], "related.user": [ "inima", @@ -766,8 +766,8 @@ "10.18.109.121" ], "related.user": [ - "hil", "tatn", + "hil", "pida" ], "rsa.db.index": "quip", @@ -817,9 +817,9 @@ "10.63.37.192" ], "related.user": [ + "reetd", "iunt", - "equep", - "reetd" + "equep" ], "rsa.db.database": "aliqu", "rsa.db.index": "mipsumd", @@ -880,9 +880,9 @@ "10.47.202.102" ], "related.user": [ - "run", + "ice", "ntor", - "ice" + "run" ], "rsa.db.database": "ite", "rsa.db.index": "iquipex", @@ -942,8 +942,8 @@ "10.106.239.55" ], "related.user": [ - "itquiin", - "serunt" + "serunt", + "itquiin" ], "rsa.db.database": "itame", "rsa.db.index": "oluptas", @@ -999,13 +999,13 @@ "etMalor4236.www5.host" ], "related.ip": [ - "10.125.160.129", - "10.53.168.235" + "10.53.168.235", + "10.125.160.129" ], "related.user": [ + "one", "abi", - "ione", - "one" + "ione" ], "rsa.db.database": "sperna", "rsa.db.index": "estia", @@ -1066,8 +1066,8 @@ "10.227.177.121" ], "related.user": [ - "liqui", "tasuntex", + "liqui", "iduntu" ], "rsa.db.database": "rvel", @@ -1125,16 +1125,16 @@ "process.name": "laboree.exe", "process.pid": 6501, "related.hosts": [ - "", + "xeacomm6855.api.corp", "nsecte3304.mail.corp" ], "related.ip": [ - "10.167.85.181", - "10.98.182.220" + "10.98.182.220", + "10.167.85.181" ], "related.user": [ - "fde", - "econs" + "econs", + "fde" ], "rsa.db.database": "equat", "rsa.internal.event_desc": "orpor", @@ -1189,9 +1189,9 @@ "10.89.208.95" ], "related.user": [ - "iciadese", "icabo", - "sintoc" + "sintoc", + "iciadese" ], "rsa.db.index": "eni", "rsa.internal.event_desc": "rcitati", @@ -1240,9 +1240,9 @@ "10.72.148.32" ], "related.user": [ - "uteirure", "tDuisaut", - "luptatev" + "luptatev", + "uteirure" ], "rsa.db.database": "uamest", "rsa.db.index": "uae", @@ -1362,12 +1362,12 @@ "tnonpro7635.localdomain" ], "related.ip": [ - "10.192.34.76", - "10.213.144.249" + "10.213.144.249", + "10.192.34.76" ], "related.user": [ - "lore", "temqu", + "lore", "iquipe" ], "rsa.db.database": "gnamal", @@ -1482,9 +1482,9 @@ "10.143.193.199" ], "related.user": [ - "tqu", + "niamqui", "quid", - "niamqui" + "tqu" ], "rsa.db.index": "inci", "rsa.internal.event_desc": "eroinBCS", @@ -1533,9 +1533,9 @@ "10.65.175.9" ], "related.user": [ + "umqu", "ritatise", - "essequam", - "umqu" + "essequam" ], "rsa.db.database": "ender", "rsa.db.index": "entorev", @@ -1589,9 +1589,9 @@ "10.205.72.243" ], "related.user": [ - "isiuta", "tatn", - "umdolo" + "umdolo", + "isiuta" ], "rsa.db.index": "proide", "rsa.internal.event_desc": "ameiusm", @@ -1633,9 +1633,9 @@ "10.107.9.163" ], "related.user": [ - "sit", + "mac", "mquisno", - "mac" + "sit" ], "rsa.db.index": "sit", "rsa.internal.event_desc": "tdol", @@ -1677,9 +1677,9 @@ "10.80.101.72" ], "related.user": [ + "asiarc", "umSe", - "quidexea", - "asiarc" + "quidexea" ], "rsa.db.index": "veli", "rsa.internal.event_desc": "quatu", @@ -1728,9 +1728,9 @@ "10.39.10.155" ], "related.user": [ - "urExcept", + "ptass", "aboreetd", - "ptass" + "urExcept" ], "rsa.db.database": "teirured", "rsa.db.index": "dolorem", @@ -1828,9 +1828,9 @@ "10.71.238.250" ], "related.user": [ + "reseo", "moenimi", - "aec", - "reseo" + "aec" ], "rsa.db.index": "mac", "rsa.internal.event_desc": "quamest", @@ -1875,13 +1875,13 @@ "rum5798.home" ], "related.ip": [ - "10.226.20.199", - "10.226.101.180" + "10.226.101.180", + "10.226.20.199" ], "related.user": [ - "ritt", "rationev", - "veniamqu" + "veniamqu", + "ritt" ], "rsa.db.database": "conse", "rsa.db.index": "imveniam", @@ -1943,9 +1943,9 @@ "10.134.65.15" ], "related.user": [ - "quaUten", + "cab", "utaliqu", - "cab" + "quaUten" ], "rsa.db.database": "isciv", "rsa.db.index": "nofd", @@ -2002,8 +2002,8 @@ "10.70.147.120" ], "related.user": [ - "cidunt", "tten", + "cidunt", "emqu" ], "rsa.db.index": "eaqu", @@ -2053,9 +2053,9 @@ "10.178.242.100" ], "related.user": [ + "idid", "dqu", - "loi", - "idid" + "loi" ], "rsa.db.database": "tenatuse", "rsa.db.index": "ullamcor", @@ -2109,9 +2109,9 @@ "10.211.179.168" ], "related.user": [ - "ritati", "untincul", - "mmodoc" + "mmodoc", + "ritati" ], "rsa.db.index": "emvele", "rsa.internal.event_desc": "oluptas", @@ -2153,9 +2153,9 @@ "10.30.243.163" ], "related.user": [ - "illu", "mven", - "dolore" + "dolore", + "illu" ], "rsa.db.index": "idol", "rsa.internal.event_desc": "lore", @@ -2200,12 +2200,12 @@ "dictasun3878.internal.localhost" ], "related.ip": [ - "10.212.214.4", - "10.6.79.159" + "10.6.79.159", + "10.212.214.4" ], "related.user": [ - "amvo", "quid", + "amvo", "midestl" ], "rsa.db.database": "urExce", @@ -2267,8 +2267,8 @@ "10.237.170.202" ], "related.user": [ - "liquide", "rcit", + "liquide", "atDu" ], "rsa.db.database": "taedict", @@ -2330,8 +2330,8 @@ "10.228.118.81" ], "related.user": [ - "emoe", "tatemU", + "emoe", "itasper" ], "rsa.db.database": "toditaut", @@ -2389,13 +2389,13 @@ "esseq7889.www.invalid" ], "related.ip": [ - "10.49.71.118", - "10.234.165.130" + "10.234.165.130", + "10.49.71.118" ], "related.user": [ "henderit", - "emip", - "iuntNequ" + "iuntNequ", + "emip" ], "rsa.db.database": "veniamqu", "rsa.db.index": "atquo", @@ -2449,9 +2449,9 @@ "10.199.5.49" ], "related.user": [ - "emip", "turadipi", - "olorema" + "olorema", + "emip" ], "rsa.db.index": "ataevi", "rsa.internal.event_desc": "minim", @@ -2493,9 +2493,9 @@ "10.193.219.34" ], "related.user": [ - "uamei", + "olorem", "utlabo", - "olorem" + "uamei" ], "rsa.db.index": "nse", "rsa.internal.event_desc": "orisni", @@ -2540,12 +2540,12 @@ "tem6815.home" ], "related.ip": [ - "10.174.185.109", - "10.120.167.217" + "10.120.167.217", + "10.174.185.109" ], "related.user": [ - "dolorem", "rsp", + "dolorem", "animid" ], "rsa.db.database": "tsuntinc", @@ -2603,12 +2603,12 @@ "mporainc2064.home" ], "related.ip": [ - "10.141.213.219", - "10.117.137.159" + "10.117.137.159", + "10.141.213.219" ], "related.user": [ - "accusa", "atev", + "accusa", "ate" ], "rsa.db.database": "nibus", @@ -2666,13 +2666,13 @@ "caboNem1043.internal.home" ], "related.ip": [ - "10.166.90.130", - "10.94.224.229" + "10.94.224.229", + "10.166.90.130" ], "related.user": [ "eavol", - "etconsec", - "rem" + "rem", + "etconsec" ], "rsa.db.database": "oditempo", "rsa.db.index": "deF", @@ -2731,13 +2731,13 @@ "tatio6513.www.invalid" ], "related.ip": [ - "10.201.81.46", - "10.38.28.151" + "10.38.28.151", + "10.201.81.46" ], "related.user": [ + "mipsumqu", "incidid", - "tiumto", - "mipsumqu" + "tiumto" ], "rsa.db.database": "abor", "rsa.db.index": "adol", @@ -2796,12 +2796,12 @@ "dolori6232.api.invalid" ], "related.ip": [ - "10.214.245.95", - "10.255.28.56" + "10.255.28.56", + "10.214.245.95" ], "related.user": [ - "umdolors", "rerepre", + "umdolors", "uptatem" ], "rsa.db.database": "odt", @@ -2900,8 +2900,8 @@ "10.141.200.133" ], "related.user": [ - "iame", "ess", + "iame", "enim" ], "rsa.db.index": "nofdeFi", @@ -2945,8 +2945,8 @@ ], "related.user": [ "runtmo", - "illoi", - "ugi" + "ugi", + "illoi" ], "rsa.db.index": "eetdo", "rsa.internal.event_desc": "quaer", @@ -2991,13 +2991,13 @@ "mestq2106.api.host" ], "related.ip": [ - "10.41.89.217", - "10.39.143.155" + "10.39.143.155", + "10.41.89.217" ], "related.user": [ - "sedquiac", "tem", - "tperspic" + "tperspic", + "sedquiac" ], "rsa.db.database": "radipis", "rsa.db.index": "nse", @@ -3058,8 +3058,8 @@ "10.5.5.1" ], "related.user": [ - "minim", "CSe", + "minim", "unt" ], "rsa.db.database": "atu", @@ -3121,9 +3121,9 @@ "10.168.132.175" ], "related.user": [ - "eursinto", + "iamea", "giatquov", - "iamea" + "eursinto" ], "rsa.db.database": "ici", "rsa.db.index": "iquaUt", @@ -3177,9 +3177,9 @@ "10.123.154.17" ], "related.user": [ + "quiac", "dolorsi", - "lmo", - "quiac" + "lmo" ], "rsa.db.index": "idunt", "rsa.internal.event_desc": "usantiu", @@ -3270,9 +3270,9 @@ "10.126.205.76" ], "related.user": [ - "rsitvol", "Nemoenim", - "iati" + "iati", + "rsitvol" ], "rsa.db.index": "eFini", "rsa.internal.event_desc": "acom", @@ -3317,13 +3317,13 @@ "fic5107.home" ], "related.ip": [ - "10.169.101.161", - "10.164.66.154" + "10.164.66.154", + "10.169.101.161" ], "related.user": [ - "orissu", + "eufug", "ine", - "eufug" + "orissu" ], "rsa.db.database": "stquidol", "rsa.db.index": "imadmini", @@ -3377,8 +3377,8 @@ "10.70.83.200" ], "related.user": [ - "metco", "riat", + "metco", "ihilmole" ], "rsa.db.index": "urQuis", @@ -3424,13 +3424,13 @@ "onpr47.api.home" ], "related.ip": [ - "10.207.97.192", - "10.134.55.11" + "10.134.55.11", + "10.207.97.192" ], "related.user": [ + "madminim", "tanimid", - "mmod", - "madminim" + "mmod" ], "rsa.db.database": "tetura", "rsa.db.index": "uptasnul", @@ -3492,8 +3492,8 @@ ], "related.user": [ "eritq", - "oinBCSed", - "texplica" + "texplica", + "oinBCSed" ], "rsa.db.database": "lit", "rsa.db.index": "ritati", @@ -3550,8 +3550,8 @@ "eufugia4481.corp" ], "related.ip": [ - "10.41.232.147", - "10.61.175.217" + "10.61.175.217", + "10.41.232.147" ], "related.user": [ "runtm", @@ -3610,9 +3610,9 @@ "10.150.30.95" ], "related.user": [ - "mini", + "atnonpr", "uisnos", - "atnonpr" + "mini" ], "rsa.db.index": "smod", "rsa.internal.event_desc": "isn", @@ -3654,9 +3654,9 @@ "10.98.71.45" ], "related.user": [ - "fugitse", "CSe", - "onse" + "onse", + "fugitse" ], "rsa.db.index": "Dui", "rsa.internal.event_desc": "isci", @@ -3742,8 +3742,8 @@ "10.197.203.167" ], "related.user": [ - "iumdo", "uta", + "iumdo", "eserun" ], "rsa.db.index": "smo", @@ -3786,9 +3786,9 @@ "10.187.170.23" ], "related.user": [ - "ibusBo", + "enima", "sectetu", - "enima" + "ibusBo" ], "rsa.db.index": "uido", "rsa.internal.event_desc": "lab", @@ -3837,9 +3837,9 @@ "10.250.248.215" ], "related.user": [ - "aevitaed", "tinculpa", - "quaeratv" + "quaeratv", + "aevitaed" ], "rsa.db.database": "lica", "rsa.db.index": "uisnos", @@ -3895,8 +3895,8 @@ "osa3211.www5.example" ], "related.ip": [ - "10.147.154.118", - "10.146.57.23" + "10.146.57.23", + "10.147.154.118" ], "related.user": [ "tateveli", @@ -3997,9 +3997,9 @@ "10.154.172.82" ], "related.user": [ - "onnumqua", + "tetura", "nesci", - "tetura" + "onnumqua" ], "rsa.db.index": "oinBCSed", "rsa.internal.event_desc": "ntor", @@ -4042,8 +4042,8 @@ ], "related.user": [ "tpers", - "midestl", - "expl" + "expl", + "midestl" ], "rsa.db.index": "olu", "rsa.internal.event_desc": "odocons", @@ -4085,8 +4085,8 @@ "10.178.160.245" ], "related.user": [ - "fdeFinib", "turQuis", + "fdeFinib", "olupta" ], "rsa.db.index": "rsint", @@ -4195,13 +4195,13 @@ "nimve2787.mail.test" ], "related.ip": [ - "10.65.207.234", - "10.222.32.183" + "10.222.32.183", + "10.65.207.234" ], "related.user": [ "eve", - "itame", - "eruntmo" + "eruntmo", + "itame" ], "rsa.db.database": "udexerc", "rsa.db.index": "volup", @@ -4255,9 +4255,9 @@ "10.16.181.60" ], "related.user": [ + "olore", "gnama", - "oinven", - "olore" + "oinven" ], "rsa.db.index": "uatu", "rsa.internal.event_desc": "nderiti", @@ -4343,9 +4343,9 @@ "10.204.214.98" ], "related.user": [ + "eprehe", "porissus", - "tdolo", - "eprehe" + "tdolo" ], "rsa.db.index": "abo", "rsa.internal.event_desc": "ecte", @@ -4387,8 +4387,8 @@ "10.223.178.192" ], "related.user": [ - "etc", "evel", + "etc", "moenimip" ], "rsa.db.index": "iarchit", @@ -4434,13 +4434,13 @@ "ama6820.mail.example" ], "related.ip": [ - "10.26.137.126", - "10.26.33.181" + "10.26.33.181", + "10.26.137.126" ], "related.user": [ - "taevit", "ati", - "audant" + "audant", + "taevit" ], "rsa.db.database": "com", "rsa.db.index": "mveni", @@ -4560,13 +4560,13 @@ "lit4112.www.localhost" ], "related.ip": [ - "10.107.24.54", - "10.10.174.253" + "10.10.174.253", + "10.107.24.54" ], "related.user": [ "itinvo", - "uptasn", - "hend" + "hend", + "uptasn" ], "rsa.db.database": "lup", "rsa.db.index": "isau", @@ -4621,9 +4621,9 @@ "10.87.92.17" ], "related.user": [ + "eeufug", "luptate", - "tamr", - "eeufug" + "tamr" ], "rsa.db.index": "oreeufug", "rsa.internal.event_desc": "ura", @@ -4676,9 +4676,9 @@ "10.161.51.135" ], "related.user": [ - "Finibus", "accus", - "asper" + "asper", + "Finibus" ], "rsa.db.database": "litani", "rsa.db.index": "arch", @@ -4732,9 +4732,9 @@ "10.51.17.32" ], "related.user": [ - "itten", "mquido", - "llum" + "llum", + "itten" ], "rsa.db.index": "uscipit", "rsa.internal.event_desc": "llitani", @@ -4777,8 +4777,8 @@ ], "related.user": [ "ollita", - "mmodicon", - "cusa" + "cusa", + "mmodicon" ], "rsa.db.index": "ercitati", "rsa.internal.event_desc": "pteurs", @@ -4824,13 +4824,13 @@ "uidol6868.mail.localdomain" ], "related.ip": [ - "10.114.0.148", - "10.198.187.144" + "10.198.187.144", + "10.114.0.148" ], "related.user": [ "equatD", - "rsitamet", - "ons" + "ons", + "rsitamet" ], "rsa.db.database": "periam", "rsa.db.index": "umiurer", @@ -4888,9 +4888,9 @@ "10.61.140.120" ], "related.user": [ - "loru", + "naaliq", "equa", - "naaliq" + "loru" ], "rsa.db.index": "umfugiat", "rsa.internal.event_desc": "ora", @@ -4935,13 +4935,13 @@ "ptat4878.lan" ], "related.ip": [ - "10.149.238.108", - "10.93.24.151" + "10.93.24.151", + "10.149.238.108" ], "related.user": [ "ite", - "nven", - "sequamn" + "sequamn", + "nven" ], "rsa.db.database": "fugi", "rsa.db.index": "nesciu", @@ -4995,9 +4995,9 @@ "10.101.45.225" ], "related.user": [ - "emi", "uinesc", - "cipitla" + "cipitla", + "emi" ], "rsa.db.index": "caecat", "rsa.internal.event_desc": "tsunt", @@ -5040,9 +5040,9 @@ "10.2.204.161" ], "related.user": [ - "quela", + "eumfugia", "ore", - "eumfugia" + "quela" ], "rsa.db.index": "olup", "rsa.internal.event_desc": "quuntur", @@ -5139,8 +5139,8 @@ "10.151.110.250" ], "related.user": [ - "neavol", "tla", + "neavol", "pidatatn" ], "rsa.db.database": "itaedict", @@ -5258,9 +5258,9 @@ "10.128.102.130" ], "related.user": [ - "que", + "sequatu", "ore", - "sequatu" + "que" ], "rsa.db.index": "exerci", "rsa.internal.event_desc": "olu", @@ -5309,8 +5309,8 @@ "10.200.162.248" ], "related.user": [ - "onnu", "reseo", + "onnu", "doloremi" ], "rsa.db.database": "billo", @@ -5365,8 +5365,8 @@ "10.103.215.159" ], "related.user": [ - "volup", "apa", + "volup", "atatn" ], "rsa.db.index": "atcupi", diff --git a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml index be7f27f551f..4f9992ae2d4 100644 --- a/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml +++ b/x-pack/filebeat/module/envoyproxy/log/config/envoyproxy.yml @@ -9,4 +9,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/audit/config/input.yml b/x-pack/filebeat/module/googlecloud/audit/config/input.yml index 4c30e23b5e3..b5e392ee0b6 100644 --- a/x-pack/filebeat/module/googlecloud/audit/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/audit/config/input.yml @@ -34,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml index d6579aa9f47..39648636c59 100644 --- a/x-pack/filebeat/module/googlecloud/firewall/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/firewall/config/input.yml @@ -35,4 +35,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml index cf89526bbe5..f1976195687 100644 --- a/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml +++ b/x-pack/filebeat/module/googlecloud/vpcflow/config/input.yml @@ -34,4 +34,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/gsuite/admin/config/config.yml b/x-pack/filebeat/module/gsuite/admin/config/config.yml index b5c62d3657f..9ec19a83615 100644 --- a/x-pack/filebeat/module/gsuite/admin/config/config.yml +++ b/x-pack/filebeat/module/gsuite/admin/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/drive/config/config.yml b/x-pack/filebeat/module/gsuite/drive/config/config.yml index 5f1bd6ecbf3..92e464164b0 100644 --- a/x-pack/filebeat/module/gsuite/drive/config/config.yml +++ b/x-pack/filebeat/module/gsuite/drive/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/groups/config/config.yml b/x-pack/filebeat/module/gsuite/groups/config/config.yml index 46a3ed338d9..ba6890c80e4 100644 --- a/x-pack/filebeat/module/gsuite/groups/config/config.yml +++ b/x-pack/filebeat/module/gsuite/groups/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/login/config/config.yml b/x-pack/filebeat/module/gsuite/login/config/config.yml index b501012b3d2..95ccc04ed2c 100644 --- a/x-pack/filebeat/module/gsuite/login/config/config.yml +++ b/x-pack/filebeat/module/gsuite/login/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/saml/config/config.yml b/x-pack/filebeat/module/gsuite/saml/config/config.yml index 1e703737e0d..8d1c7645fda 100644 --- a/x-pack/filebeat/module/gsuite/saml/config/config.yml +++ b/x-pack/filebeat/module/gsuite/saml/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml index 773ab620173..dc9e3d353e6 100644 --- a/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml +++ b/x-pack/filebeat/module/gsuite/user_accounts/config/config.yml @@ -39,7 +39,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - script: lang: javascript id: gsuite-common diff --git a/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml b/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml index 2130bb419d2..bca14ecccaa 100644 --- a/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml +++ b/x-pack/filebeat/module/ibmmq/errorlog/config/errorlog.yml @@ -12,4 +12,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/iptables/log/config/input.yml b/x-pack/filebeat/module/iptables/log/config/input.yml index 6183661122a..bcfde17c3c5 100644 --- a/x-pack/filebeat/module/iptables/log/config/input.yml +++ b/x-pack/filebeat/module/iptables/log/config/input.yml @@ -55,4 +55,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/juniper/_meta/config.yml b/x-pack/filebeat/module/juniper/_meta/config.yml index be40af66202..7f992656788 100644 --- a/x-pack/filebeat/module/juniper/_meta/config.yml +++ b/x-pack/filebeat/module/juniper/_meta/config.yml @@ -36,3 +36,16 @@ # "local" (default) for system timezone. # "+02:00" for GMT+02:00 # var.tz_offset: local + + srx: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9006. + #var.syslog_port: 9006 diff --git a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc index c59b7ac4a95..3e145ea81c9 100644 --- a/x-pack/filebeat/module/juniper/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/juniper/_meta/docs.asciidoc @@ -5,18 +5,131 @@ == Juniper module -experimental[] +This is a module for ingesting data from the different Juniper Products. Currently supports these filesets: -This is a module for receiving Juniper JUNOS logs over Syslog or a file. +- `srx` fileset: Supports Juniper SRX logs +- `junos` fileset: Supports Juniper JUNOS logs +- `netscreen` fileset: Supports Juniper Netscreen logs include::../include/gs-link.asciidoc[] include::../include/configuring-intro.asciidoc[] -:fileset_ex: junos - include::../include/config-option-intro.asciidoc[] +:fileset_ex: srx +beta[] + +[float] +==== `srx` fileset settings + +The Juniper-SRX module only supports syslog messages in the format "structured-data + brief" https://www.juniper.net/documentation/en_US/junos/topics/reference/configuration-statement/structured-data-edit-system.html[JunOS Documentation structured-data] + +To configure a remote syslog destination, please reference the https://kb.juniper.net/InfoCenter/index?page=content&id=kb16502[SRX Getting Started - Configure System Logging]. + +The following processes and tags are supported: + +[options="header"] +|============================================================== +| JunOS processes | JunOS tags | +| RT_FLOW | RT_FLOW_SESSION_CREATE | +| | RT_FLOW_SESSION_CLOSE | +| | RT_FLOW_SESSION_DENY | +| | APPTRACK_SESSION_CREATE | +| | APPTRACK_SESSION_CLOSE | +| | APPTRACK_SESSION_VOL_UPDATE | +| RT_IDS | RT_SCREEN_TCP | +| | RT_SCREEN_UDP | +| | RT_SCREEN_ICMP | +| | RT_SCREEN_IP | +| | RT_SCREEN_TCP_DST_IP | +| | RT_SCREEN_TCP_SRC_IP | +| RT_UTM | WEBFILTER_URL_PERMITTED | +| | WEBFILTER_URL_BLOCKED | +| | AV_VIRUS_DETECTED_MT | +| | CONTENT_FILTERING_BLOCKED_MT | +| | ANTISPAM_SPAM_DETECTED_MT | +| RT_IDP | IDP_ATTACK_LOG_EVENT | +| | IDP_APPDDOS_APP_STATE_EVENT | +| RT_AAMW | SRX_AAMW_ACTION_LOG | +| | AAMW_MALWARE_EVENT_LOG | +| | AAMW_HOST_INFECTED_EVENT_LOG | +| | AAMW_ACTION_LOG | +| RT_SECINTEL | SECINTEL_ACTION_LOG | +|============================================================== + +The syslog format choosen should be `Default`. + +[float] +=== Compatibility + +This module has been tested against JunOS version 19.x and 20.x. +Versions above this are expected to work but have not been tested. + +[source,yaml] +---- +- module: sophosxg + firewall: + enabled: true + var.input: udp + var.syslog_host: 0.0.0.0 + var.syslog_port: 9006 +---- + +include::../include/var-paths.asciidoc[] + +*`var.input`*:: + +The input to use, can be either the value `tcp`, `udp` or `file`. + +*`var.syslog_host`*:: + +The interface to listen to all syslog traffic. Defaults to localhost. +Set to 0.0.0.0 to bind to all available interfaces. + +*`var.syslog_port`*:: + +The port to listen for syslog traffic. Defaults to 9006. + + +[float] +==== Juniper SRX ECS fields + +This is a list of JunOS fields that are mapped to ECS. + +[options="header"] +|============================================================== +| Juniper SRX Fields | ECS Fields | +| application-risk | event.risk_score | +| bytes-from-client | source.bytes | +| bytes-from-server | destination.bytes | +| destination-interface-name | observer.egress.interface.name | +| destination-zone-name | observer.egress.zone | +| destination-address | destination.ip | +| destination-port | destination.port | +| dst_domainname | url.domain | +| elapsed-time | event.duration | +| filename | file.name | +| nat-destination-address | destination.nat.ip | +| nat-destination-port | destination.nat.port | +| nat-source-address | source.nat.ip | +| nat-source-port | source.nat.port | +| message | message | +| obj | url.path | +| packets-from-client | source.packets | +| packets-from-server | destination.packets | +| policy-name | rule.name | +| protocol | network.transport | +| source-address | source.ip | +| source-interface-name | observer.ingress.interface.name| +| source-port | source.port | +| source-zone-name | observer.ingress.zone | +| url | url.domain | +|============================================================== + + +:fileset_ex: junos + [float] ==== `junos` fileset settings diff --git a/x-pack/filebeat/module/juniper/fields.go b/x-pack/filebeat/module/juniper/fields.go index 6122a564654..e22907d0244 100644 --- a/x-pack/filebeat/module/juniper/fields.go +++ b/x-pack/filebeat/module/juniper/fields.go @@ -19,5 +19,5 @@ func init() { // AssetJuniper returns asset data. // This is the base64 encoded gzipped contents of module/juniper. func AssetJuniper() string { - return "eJzsvW2TGzeSIPx9fwUefzhJDpmyZVt7o5udC213e9w7ktyrluSNi4moAFEgCTcKKAEosulf/wQSqBdWochuNlBs7d18mLCaZCKRSCTyPb9DN3T7Gv1RCVZS9S8IGWY4fY3+w/0B/cen979d/wtCOdVEsdIwKV6jv/0LQqj+DVowynM9+xfk/+s1fGr/9x0SuKCvkaBmI9XNjAlD1QITOrN/b76GkFxTtVHM0NfIqKr7idmW9LXFcSNV3vl7The44iaDJV+jBeaa7nw8QLf+33tcUCQXyKxojRhqEEObFVUUPjMKLxaMoBXWaE6pQHKuqVrTfDbYn9L4HptZKlmVd99Kn6jtsoC1wHxne+Orj60fWqJdpNDLnb/vX2H8wAan8nHFtP0eYhpVmubISERwaSpPf4U3qKBa46X9NzaIyIJqu2lpP++BRuitXKJzSmQObBzYiIPF+kgdu50aLl1TYTK7tciAPcKJqe9JroHmRApDhdH2fjChDRamRkMHcTSsOAbBHJv+B0PsmMPJLoGwQZsVIyuEkaZaMynQihmNMHpPze/MCKp1ffqzAWs0m9UrWfEcCbqmCs1pw3clVpqid9RgixpGCyWLzlJP38qlfnGFyQ01+tkA/DlTlBi+fY6MxxujD9QJC8fhooPmLEhITteUH0FJLkX/fu5Q8pyWihJsPCY5XTBBcyQFB7QMnnOKClyGsSr0Mot2Yfac8Tt/zy/Pf0BrzCt/41lOhWEL5rmT3mJiEJdLd15qcBCwO2bBe26B79njKLEyjFQcK/i9P9jZKGcMQB/FKSHOGEAe55TRI1lPeyYv/9+Z7D8Tu2qaA3nY9ZXzPzLYSP9YHg12a3yM0EuOmqJaVookensfTrZU9/9hmGmDDS2oMI8ROVzlzGSE494dfiToUWHU9jEitrI61WNEjInjEEurMdWS4/FyWk7xMdIjLdkWlOYxbagRvSZkZ3a+WLsFLDYDPWSgJDzMiujpIQPoB6yIcSr2XCsTUVF0vCpB8jlyDbYZiXwoQMF7k49MoVZXgn2paKtGq2b//k/bXaP2TApiHwds5GO3bEfEzZqlFYdd6p7ZZdiCEdy9z2/lEl2sqTDoGoQzqkROlTVBFPWCarD1BbulOdLUWCA7P95dQ48bLPUhDGA/2GBpDmEA+l6HMvQExvcvHceYg33dgyb3o8FK6kT6apcvf5XadEUk73OkpiJnYll/qENs0/EhfT30Zccw2OBHo4S9vFr/hHCeKysrx657n7iD3Rv5tRJ3/So1eV/930teS630sqEvF5wjrestyxFGS7amonGSfb2KgCXRcf6LtBZI/hiVv68jojHq0JDlNlP0S4Kz7gYP4YBh3/MtUPnCLY2u4CI9995sg9HHbUkRwUMJMqeIMrOiCn26FOaHV0gq9AuX2Pz4Es2xBi6qA2QLtqwUqH4H9n2MuvsV7xvCoOmMzwj+BfvrpUzlZttnHdcrf/UOBqk2WOXJlLqOROtsu0vJy6vPO/oeRopy3D9ShPRWG1r4R9SjbaGtqONU7Yhn/y0VWzKBef2bXW3lAB1S6V97EiMurz6/CpDAoz+gxMNJ0GA0pHKM16dl1KHieOzrs6I4p2qS2PWvsBS6PH9IlNTh2w2WApjjYqWP2snGSZbcz4ZrReuyVbTgoljT5UxyTomR6msUwJZ6J8i5sTzHNCKOdDS3mO4oqm9lX21Bewj9CC2+gswfi6paSA3JboUUaL4dHBpCin6pqDYWoGZFybf+nOyXraBHFJMV0iyn6On3yKxUhV7+/PMztMEaaUpFs8oeSjwK5fUOlNClFJqmIwX5ariCyEqYxqdQFXMn9OxV1kEI6CmeyzXtEIOJYGZlLd60URQXo/eHfDVsc2JS0ZxVfT0tBqG+CWmOjWOBLRAz/6xefv/DX7QT6S9KEKA10v8c7Oaf1h58i7dUoZfoQhBc6oq7yIo1Ke8l10PQHxj8CORWhlb58SX6N7vd5+jHH9G/ISKV1ZdhF37R5+h/cPO/7BeZRrtE+SZ4hELm9NHaumJDM4I5n2Nyk1YDdsgJaeDaYOPsCktEKvJSMmHANDE0nOAMzJFRpWSi/LRWH9QlJQxzwBgw1UYqq1mLrdM67AdrzFnuGCOEFEILWYncvjCcAvJMLL1ydDB5cfdGDCDHiAX667AnbDRyClsucf5Y3jmPDtLsT4oKahQjAavDm8LdL4Mt7J77WgjbZx+bVqOVi/rYZuhXubFHM7Q5mUBSWWPMSHRDaXmAaI/ixftKiKYkoVpna5Zneaqo60UteZZUUIUNXPLcUrBjF66ZMhXm1mjf8b2LgIuDFcya3RArB2K4XfirfnmOlJXWGhwqQDSsltQ0XztICa0SJT2dnBIuE24/JVSSUNBQ8F+e177XD7SQhqJrz+9EUXho59sxQWn/VwdivoLAi18p0yVnKTMbHrU5r9lA7X8UupmVuQn5HW6dfQM8r9dcV1st/gn57xFhdOJlwfgJYvR2VWscXZ29ufK6L8HCkocVpVR9jRfBE/nVpUFUj8P98ck9VWCIg+kecqXumvJV+5PWYHd6DljmM/Ty51doA3QvKBYIcx72FYBTH9Sk1n+ENlRRBxYbxCnWBknRKxfZJeLJ1cSvm4iBu5oibOtp97tUORAOspooWQnJ5XLbD8QtmBposQj9jMgKK0yMI6K91FvAH5zmAlXC5/TwHZ/5aEVt7IJuF6hPGUTYE7sEi6KwSqYUdRhB4c2oTAPJ2lMrMQGN1cUohPc5SEIqVUPUBoscqxwJqQrM2Z+h/F6piiB9cp/lcDSJZDUfPEn3IlKLdYPMC84WFHYcMPA1JVLkIwp2e9yZNin9LHs2xASRRcmpCTLAqBMVgwJvFOuJwU69mTInYuRru3aQncdYeZczR9mvkMKsIh1TW58aK+elzXLKT0T4C5GnILsF+acUqbst7BGLdvVaxXTptR/7FB6IqGQ3+g0y9Nb4y4fWVOlOOUW+Lw8scL4PZbYtxbG22ZbpEalymqd7B32SjX+mdLNirWPUmTbNF7vx9eFrpWQxA6gVFOVrQgVWTDq1vqi4Yd8ZRhXCZcnr6pe2l02BBV6GSnMR4hDeqe1Fh5TDVSNmnmgkN8JFxgwuyr5n0GNsV7MoDm+f0YismLVuZE71DL2rtAEzqQvU3kpsRvJysaFHHtJeAbZYWLzXdApNCA65XtDRTtEFVVQQxxDYqtY5W7PcajbAD2FBdl0Lso894oU3eVsyNdkO2/N0saBby4nM8K3brLZCz+prFilg0P2+0YiHPurCeW6lcSPPZoMlm3QyWcWWQMVAkXsoxIb+sa8KaJBfKlpNxkqWux0XtfJxgzUCJPIRvgHkfohN1IhKwQ5BE8i0ZWESvL7LIgWuZZYA1TJLoT2XMUXRLtCX0aEm0JU6r8hpTMie+Rh8YwbP5b3enGPF5iG5dkywoH0get0QYjuCMBko8TEUa13x1GGnEStKVobIgr5wODTGC2Rly8WAQ7DwJNgxIEcYhK6pYiZl6ciejdWr+yLATmRnn8snbfHioHege6WbShcLDeJOJSVswVrDJ6zdumDOWE8Vryunz2YKHEDjYmR5WzBRu6hyH2QJ4u3N5qkO4fOuld61BKVCv1371Fim64SAvl8N1q9PaKxKUpdSs4iC4068Bea0yF2HKUjlr+/uaBeeipssXeuie4oiURVUMXJfWRTc2wRVbHs21q1ka26GE0vufg+2tqYil8onzO7dmZz/cYLuNXVoV87/oCRsR1vE0teCD8htJeh+xJykT9mr7pvhhfRV/17MeC/XCje5xUIahNHKd7wIJ9ByuczqRJWTCPWaEe8t1KfombIj+/4O6VbQtRrER1jxl5yRberbs0cuXAECvrm24NsRuVzxlHnTYQJ+qDgFxMLiVApDb1NrrA1Cl8L569p+qDjPtf0/eFQxrxEKNYA58DiTFRZLmgm6SS0LxgKXdNMJ9YMSYoxi88rQjoQY5uhrh7rV1rvPX1h06BJHE3YN5ThL1rZyH9HAEOznFzlkuvpbwLiFCjBLsLrhoG5zvtSaqhm6pu5QKk3VDC8ptPL2me4LqWocBrBrME5vJ/B75H7f6VshFZorubGf1X/1uqYzu0b7SV/mV1iZ2G66BnBsj4q/U3JQHTrVnZI8b9TGVFdKltQHFFO9xW8Ewpwq02QXqXZR/zcX3vLio9MEAJKQAgpzjoQU3ylaUrBk9mU/gNkw5ZNDKqXshWnsFThJ0ONeMBdhq8M/g51tmFl5ZdnJenQOC86h2kQgKb5bSvvfe14CUFKygOKYcN+4Ewx8AQhYJOUCWelgGNUzdN3KlP5gg25lVRqMz1w5X6WtEeNKRl2yTe7Fryc8RoRX2tQM6f8xOCb4CdP2JH1NtPdvWMUXPh1XgSbXftwNC1v0ri1TOqXsySHDy2J5DlggrLUkDPyl9jSC9iQc2Ft2Q18jjMrVVjOCOcqZvnmOSgUzUZ4jasiTsKKMFT6m9vKeD72rs1G4oIYqjUqsoYuXhkYOrhcBkUVhpZjcCdoPS2uoIXvVPfcenErj65xhgofJiW8ii7Ia3sEEx4bRholcbnw+LZGC0NI8bzIpRokx2Oai4nyLvlSYO+dnLgvMhJcaorMQlyNPV9frGUtd2rN1qxK+ZeKG5r4WqE5Exxq8U95AsZ9806A2Y/m+g+ODrhBJRV13spNzS/QRqNGDkVYnweu30nte0fWwXU8TdKaqYP3BTqldrH5NwNbx/35N+8fImvaC8fR3vNnyL7Bac40VzStCUR05omF3m6aKYZ4FXtNkj8g1LFmrzf33sfMA2hdm1C9AyY0+quVADI+xX90+dCusV80NtWphoMqwIiuX+VvX2DRlhmc1pF6LMLuRZpmZVsT+qvn3sNIUWXkuEIOcu0oQTrGyf4JGeC1qvoDQeztVXdh5OPrghF817PP0qF8sIos5E03f7O6D5ctG1T1erzVTlZ7a09fVRgCBcY/fNAHSwJU4c6u7nozjnlJnwSV3jTfkc17my3P03kmap75xA3LT9nzRr8XtWVivdg7oU/jyO+7ny3MgqS95a8TE0HuwG5FzaYBuCzPHRFYWbJgOG6lrvU3Zy343qusLtJ26sNePLZzxPSHXWNKfNQujy/ODmmws/9wBTdYi9lLkrUY7Q2euPtP3O+Xug/3aLCCodr/xwzfeHTevTFO5KU3zGFWCU+0oI92DspFojRXDcz6oAnRNGZhAJccjgkBToZP2R9k50K6q6laeWUllNYy6vpDZc75+cXnV16GRbxnrPApjddlHDhS8cy1kG2lxSKJLYdA1WwoMwmKERUupUjavfTKQX5ZJr2rdTUJXR/hPi0jnLgOX5TLAOO9/+4iYILzKqRVnfpCt/fkMPb24xUXJ6Wt05RwiDixI71nYLwKRucljm+Ccap+WMGZM31iV+wi87lGK13FjvvdPwwemb/aEXI1iyyVV6UbYhUn2uRsL8DiAdrpSVK8kzy33OFt9ZNLoTuh9As/CMPbupfLTD07HeNY047g8D5eR3Dk6T2RRZhPnXcGp+NwrGOPq/Hu6mn9n0ZEC6lMXMG5G5hUZs9K8WnqirLEu5o20lAo6D1i5XuM3MiUOq3yD1Wky9IZd9a10xf4hspsYaY381ApRjN5hUvdTDiu3VgRNasdI8V2toKr9UsjZmtGHWiuKdfTcYG2wqWIpzo0/CjN+MrPDLj6Xt4jlL8bfL/uyVlNgaDH6NGh87O6CxSJ8det3LPH0vQGTnw/n7h3znDEhq1gxzk4diV5Gv1NWksZ0Ogw8sj9FBpy6M+MOS7zh3Mo9pCtCqNaLiqMLuz4iMqfaskTd7DdsWTCR09vIBOBMm+M0zwfKFlgYTDFVIzGnCuKbBVaMQwZPwIPn4u9iiTAQ8Tv72+DORAI+lHPXXOhEGrFfHT1t8jlLqnTpi26dhBmQzKsIbUJ83eHp2UiRoXNzDd/j1AklTvlqkry8r8p9236ImdAopwYzHnAyzGVlOr8b2Zrkk+dm1h5b3OSxAR7jD6mhRcmTZfO8QTldYB8C8p0v6xi+z9a0WvGaKo63UMhlpH9c0dPAjbQfgNXtf00XdRW489Vrw0wFjRlRcGOtbTBs2PTQ6xo1itXx7xAcG9MEsorIorD3KQ0bnTnoiHWSfUsl1yx3/rO6i1xB9WgiVC7J8YHG+3vLfmG81RpJNy8vrBrclpD0dBpZX6+eVtb/IedH+p2O3t5/yLkPwIRvV8nSNc49h4Rid/LXV5focqBQddFI1rXWV5fsxyBiYVdTDbuMakjfxx/mc6vDyr0TEdlc5qkrvgYVd32lw+OCLC4j6tEqfrcEFzKYoPK84wL2pcMugbaJh7Aly5tQzogTr4htNQ7KwCO8/PGUvGbfZZXymaqne199ct1z6kAUJGvcUlJ1vQgu9WtOQ+WtdRemfYkbEzhCgl7xfNch0lRX4jVmHA8DGahxhSOor1xQpUYmLbg7dIyvP17czRsrhW8A5QKwgy35dAPNlrMRiciKbF7l+Ta6f4YVWdQ6oA7cStPjGp3v9VLFh6iYjNjloFdil+lqioIEprvZq67nKq5yZprKurYvmscoNNiurdhwoqQNL+zfpMsSi03B9WRW+dnnC/TU10p8rrjVleeMQwEH5IFd3JZS228+Q98NHQ2iH4W5EXIjdgwhTUkFzSzWu9BHJm0SPIELrp8WelZXub/3pUlv6RKTLfo0aq5xNlf4FEX5fuEdEjOBCszEQuGC7k3HKLGCqb3p+yTsKJdXsCx6L3OXHN22BexknQWQQge0L0gVsIRIZSHt9o17Tzfo10qAKflO5pSjp0ysZ98+R0yS52hu/4/a/8MC861mevZtOL5oSJktOB5Mzo+tQ+1q+GdXCBYFXxfIyW09/Eou9jZqMDIppu6vc49n3QZBU2UZOYjQuogrd3uYfX73O1YUfXQJwN9++/nd728+XHz7rcu5XWOF2ShPbqS6iVmyfPCC/V4v2I2wjTrBsIitRPianbhdSprnABP7XGwTmDALqajQjMQUIB1XUgKMi/hekEB8IBbQbIPZcDjxg70D0Ps8NlB7fWKXqOtqnuhSmHmujYpd+Q712skcYt23NNo7Wtd8pHOSHlvs0g4GG6g0vtikrXvx9S4WxIKNOprqrSZzxB671WA3osA2++U9YaF8dD/B+zsuLPJe//8wXLVVmd3kv5OwWN7x0XtE9iJ5Euao47j78JNygqStnZPt2KVPTZPRXmfZQZ/MZ+B2G3Du4ch03bKaTREPg6KvBWbc0rpu5nLlZcblebe2DTpxWXPQ0GWghcF4VmGdc51ZFfGI/RyTeA3p1r766EwWRSX6nqgBduK4xk0Pxe49vTV/p2GdusFNH6dZPxS3ayzyf5fhqFmLm8GGHSMZHozdcOEd5HSlS0aYjJYlOpUFD9hvsBLDoMNjR12LosxkKmF8/f7dFfrN+VHbpNQwIl8mTSW4/s+36EtF1Ujv1oqLTNF+p860yQ0dh+gWfaiLzoJpXY2WTiI+pF2gMvYYAQu0PMpxdAiqCQTHHgw3jz+gAXOsigSnZcEmcC/gMmIBcgO0yqNNpd2BGbfb1Q7oHJu+VvhQuHMqyKrAKlZZSQN3W+LB+OIHR58wGaRTRYGZraLzAqGLuAVUDeDFElotJQAr538kgFri6JMwXMep6OwFQfeMxX5wfOe2glrVMzrSIsMEBqPELz+xsLWIaLx3AM+X5foncWtW0d93IjJiVJbrqH3XO9At5OMiT3cAvOY4usQQGRVLJiIWRQ5Bp8iNFtki0xtmSHT5IbIFlxuNi/i5K13YwqzTQU8QdSEiYyKlOGGipKqYb6MlvA9gl+QmDfA15il4hZVZqaSRWfyQFEBf/5SBxzE+bJ7sbnK5zPIUxLaA4+e/EZEV+DYzJpbbYBew5WhOEzwKBROJkGYiHdIl1xmf8yx2WHQH9vcJgUfvDN6BHbsXYhd27KreLuyfE8J+lRD2vyaE/T8Twv5LGthGlhzPaQqR0kCPb56JrKg4KN/zbYJ3sgZe3iTQS4qKs2VRptG+rZaJ+TJ2EpKHzFIoJZp+IfF9IyLTLiExwQlqRdJYkxZwGmtSb3VVJphFSkRTVp3EVDXSWNOD3iYQIUYaa5ilgg1mTRLglWC3AgupKUnAhOtXliqJHoX1K1maFcV5AreaLMqM8AQ+bAs4QZAE4Kr51sR3i1rIOgnkssoSxDSIYoYRzBMUEOkML6kg24hZV13YAvPtnzSfp8B7nUEb0CSQXTuYNFi7xNok0OfLcv0qjQ9aZ3Nm/pKk0RjRWdxZcT3ASkYX1TrJNQeolKj4VW7a+fijzdrqAKZm5fz88Z0jDjiofUmAu27y8TrIdWAvGKcpbBidLVIcIlvELM7eBZxCN9AZKyFJMUsi6li5/inXphw0848EWyuSBDZnC5rCjNHgaC5ozqIVjO7CZiINlxQyrzjVRKagtgfOlglkkyz1BpuoM/870EMZ5FEAK7pk2igc3xPSwk6g8SlapiK1SkZrDZ3IVSL56jLzHYsngG4UxUUCRdKVAqVCO51yvVlJpjM3YTY+9C1WOAmD5yOFsDEgr918+9hwmTZYRJ9znGszr1SsYYE1VOpmBaWAWkXHNb4eXdckxwYLkxsW8YddH9tpYB/MJc7z2HeA5bHDqnXroARvESsyoqQsknQlsoATmGmsyNIkR/qORynIXN5Eb89U6vgtS1mpS8UiA+XYMFNFzz7jTNB4LXZaqDrqRJ0GLhTfxndrcem6nmYLLqM/5w3wBCn/1uaNLnUs0AQSx9rQCVCNnpvA5TIJ64plkgtcShVbgBXzapnimhVMkxRiodBJGDbFHAhBDTRXig43ugx3DaBjZ/w5qLHT8cRmE9sCSVJRJt0A6OiWqIyvGUnFlllgHteD4W4EVfHfrDJzQ3mjg406mboF60a8JmGyBIWbfiZObGHgwcaWBmXmHEnR0cVa2w8zsopV5z8ATW9LFj0QUFJVLBUWZtBzNwbkTRLA8Z9e14ns06feFNAIgJVcZliXEQcGdEErHBuqopin0O8UJUAH13U0EfD4RLaQ47Zw7UCWKk+AcXxHpk7gG9bON5wgH0DT2IkAbuBxAuNE0y/xGSDUoDUa1ASmlGbLBIJXl7G9bFqRFPdAkTy6Iq0VCXXFjQDYxBux1YVZ6ehdNddExC6UCE6LfShQ16Qz9vbN0sRnKwc0fkSvmekZG+62jN6ttcrnSfLQK8UTvIWVpirLWeyq9yRjK+rIUAoyGKINLmJ7g9cZE9rgRQLNYM2USaGGr0uRoHWTkaoSMd2sobZogY6ibyoj0YdKoMHSTfZIwmF5nzFnOTpTNGcGnWGV+26GGtq/h9Fxk7MSUmlsQiiAgSH6CPobEMlRqFSnyYdgIh3lLoqSyy0dDBY8SL+FrKI19b4jj1kaOp8RzDtTdElvUYH7jRbaWKxYVv1hIMmR5EzDcIZ6dX/00EAJ6aospTJo2HgUoc0KG8QMKhVdjLHCA9Jy7zOEIkR4b3U0KCAmfGf3kb7QnInUE/k7qNrVunhqZOSSmhVVs/b7eiWrwYuGkKBrqppxREaiEitN0TtqMEwEd3cVNyR4+lYu9YsrV/b6DJ37EV/PkVkFphRBM+AP1I8+BrQFek/N78wIqsPnPGTqJMRbwMju5hbB4m6zmmJFVjMmWBA/mLk7QX/tnviEWRiQDPGC40rArN9lBXNc6ybu4QbuvX7te/aUvh13s6emCbefXzxi7NuDyCLWNN2t8yosiz7SWwO3YsxdMMU06hGB1A6uew8TqgUfmXgJ3XMTjgOH/rmaGqTol4pqs6dp9/HZyvfvle9UBhjL41Z1ErvvkWryTnfdKftwchhBbGzn79ChXb8O7jzm7P/D8w3tYpfntVCAtcO8AVZDvCTee7KwfVzmWFPk0rUbbNDgVjWn5H9xGnxFMwq+wVwq174+SEaEsEaaUhh3hvfPq1JYaEwmGO876DDtlhag9rZMQyoFE9D2IV1SVTCnbkyFdLukG8zB1ozTJUWcrilHWGu2FO7g2nn9YdaHlswnlN+w/h5On59k0rPFrBLsS0X7YxJx+PJ18D2uY+JxU1BqjYbl7kISKQSF3Aq0YWY1JigQClSGNBq7okeVF93btLDkBHnSPFFcLhnBHFkMRkwfwOK02MFSI2MaT0e7crXVYfQ66Wwb2ctqjf3AY86wzlYyuU3gjLjGXINZKu1QIysVuyN4wv0AkLs0Flt40/wgFsIpVrM3XEtriO/ct3MIlqNf/S9m6I3YNv8aQDdgy2thEM5nRBZlZagKi+Ekbny7sXTm2Tf9s4AZizsHwsw/q5ff//AXa/ued46jptg3QbQ9n2ZxI2Z3ddzgLVXoXxufnH7h0QDkwrc+dv1Pep4XLc47XL/3PI5MXj4k2570B6bYdWbo/W8fL+zeqaLOeQL+0pxpomiJBdlardKrZ7yfC4KAQs/Rx3ev0aUwP758ji7fn1/812v06VKYVz+hp5vVFgnKzIoqRFZS+1FpUilKDHzrh1f/+/979iRIEWpWCWVcnx4gU2cFDo/j0Ym5757X/Nrx4mWNVPiK548L6a5sOoD5kQ3j7vzAh/DtKaatdfKZKVNhjt6+eR9E9k8paDpf1nGc8X+koLMwbS26X40IhY0cFp5wBI/xDd5zDkts6AafYEQ6cPcVepPnCvy0jstD6DRPLynKY+OcD42FXJ69u3Kv0mh4rMB6wujHjlPJaar+7UaXVxaVEe+XpeGRkyCi0NCuPU7DWhPL3HStaQVEB12c58x+GfM2YNuZ5R9+5yZkAGsSwgWX/oaf77LAAJU21zqJXnfXJw2j9x7DK6lMI5IHQjeHABscADPbw5JXT0x7tx8mlvVjUm/r3RjhBQ3ZjVN5cT12YPlirSVhVuV0fqOBjoOsXFZYLOmsMZ2IFAu2rBTN0XwLMKnIIWsoLGfKI1sPDIpGR7Tl4KKLBP0OeETdv1vCFd0BoGghDc18Znf8PKP4pM2FznDmUvETgC6NSgN8kYAlFgmqhXmK65Cq/0mZgKg4z2pPXDq1vG/B233M+qt1nQkn0GAvzIoqQQ36uC3pc/SpfsbeggPsR3RVO8AGL8FvY5paPapnAmVixDSukfZ+8ecIcx5UJsr2i5DghhUk5q2psm8gE0YibeAxZwJ9uhwVKAQSZJPJq+gi2wKVZYKxbxawojp2Rq8Fm6DExb2IsVPRwd+eAFs3WiHjVCyjT4oEnK3ykVALHdFAncqDeScAIxCBdIIFwugXqTZY5cM53Qi9WUKyl0LY3vhbyKWbU7OhVIRVz8hdE+8b45YG826oziGDoGU8ZEYMdsiEz3OFtISCGSuW/IiN8BbXHIsp4vh3cFDWCSIdF+Vgg7suyzaSsrYW7BIM2N2XJ3akkhLoQrCO1w/ubhF7rAwjFccKQb9oVCPx9OL29Vu5lItFePo7JZlZ0eTHu4PsR7ugu40dvC8s3hbdN5VZUWF8svgo2rqK2Tnhbgk9bslx1D9pqkYRlpUhclpK+yXHEb6uCKFaj+AMncePa452XOIJ4IWsiruUaosChQkD3KYQTjs40h6OVipBgE+XUth3xcqtkHLY/BANFKXdXa3j9aMbeTcxcl1LoWaAM5o3+/F+mJ4+zATSzFQB+YmguIB6Ee2hrrBGOJelfV3MijKF5Ea0R+YIZ/CtFLIYyauFmRyauRb10yoRVrlnIrfyRyrdEACjXxin6I1HbDYgw12cvaLZmLuTownjzf5Pkq4wSoJrn7UQlwqhPQYIEbPe/QGEcPl6175eIzYlxhNC5zJl9UBg83O6wmsmK9AuiSxKJQs2kqFIp0buQuA5hyKyBTrbjxsT60bsJESyj+GO1omCCOxgGHW4zBEIBtZv8Et9up1Xtr1vo2zXlllWwvTL2WJr9DmUgWfkGLP+TloQvMdLKqhipN4SEAQS/fqpBcys4KkNzXZDHtkZ+WGmjRoPftZ7Oqbt1sn29HL/nrx64dZKuK+gadoY4YYVVFu57rQ9RUs6GkTypxCtKcTBg4DGgw88BnVH1jqmd/fJWOvHu+3ph0xHG3J65615h/GhHQ72BjtuBcIdhMHXu7uXB3enJj07d9Gi7E0dPrlovVSnESAH5HgjQL5edvzx8JHFGm0wzZHdTT6qSSVIzDt2B/kxKTvG3NuAGRulHkrQen7q6JU7lVllBTUreYIoCd7xJCOHhv/a6IFDLyUlk3qd9kR1Pkju/bUWkT18mcgT8l+zn7//Hj19e/7m6hk6Z9owsayYXtEcSuGDuHC5lMn7Au2LhEG27MLh4Y8ZvjiSMaZkYq/ivvpPe6ohDJobAx75aEOf73NdCKT9N3W/Hccf4BSKmWIRapPeZophHqs7XW8jH3DOKu1WQFIhzQrGsXLiyYpNe4cIvOvh8iq455rlU3Ya6WbKf7KMUHsRe30x20uers7ijdh31yGs4SsNO/5f7ySCTwa84B03tFOWkYddmVKlTAwYhGyA1FItsWB/7smqFulY4a7EPoLSXZ4aIfeCqWAtaaKuP7/Y5eC1cC2+XO+inazmXynmZkWwoqhUNJcFEzhYcNcRT1fYMCqMPpgez/GUu32LT7pZ1/qRlokY116dJ1ZwlVgZaIbUbnW/WJ2w2ZEXNneRqAuaU4UNzbNoSWV7+MMKn1/qFZvg2ZWSa5Y3zcP893BZcq+pDhjDN/+xz9quThtWcNpNsnyiXTZL+l5/ZjuyzeDwUMicXDMXPV/1FfeRFnCN0hlzKPh9NU96CzpT50edSuhlYKNORwWNFWukjVRO4ltoBTUYVnsC35rZbz0J775gec7pdFLuHax3VzkXON6O3DtKztXjMabZ7pVfrdNhSGzr6OxzVHJsj8y+z1IhKojalmNefkiFnMCevEMGnWpsy1+lNugdJismRky6HCeSHN/0af1JQKZ/qagVH1Y/ck3O9Ay9zXGJPsM/nH6US+HqTv85fDzRCq+p1Zw4xQp9qajaIuhBqEspNK01qnBxqt1vBr+ZRl76HnjEQlas7gIp3PZdX75xPOstTYBqy0AffHPUu2IKU57SOsz6PF63lt5pYmRtQ//wMo1UJUTQjtXPm5fHRZ5dG6mRGjsPMfMWZvqDwGjDRC43GumSErZgxH7yPFQn6PNkhxfEbs/h2+bcoKfQEZYK0j5DELp81qEWqgS842/pEpMt+qR3G982EdiiX0gbPbvWrjCBwT7y2ndNLUAFatWAyeyLOKB40wcgUP2/U2kK5TxD8u1uO71CPdad16nXgR3DDoOM5n9zxGanyesd26rP8PWu91rWXcDWx7uADnczjcOuCRjsnk2bkOmOYXBC4YYUh4ufoWwg5kjA0Qo32HJOF0x4Xz0IJ+jqV+BypOkgYHdUoVgi3FoHTE/9iy0YG59t6r37XkojvSkbH7YxmKyKiVvgt6sCwdHAOuoeR5IhL3Mm4k0Qi3o37JahqDDt4xkQUt2yHTgW10a7Le8PTO0cYJ327TuAdYlVzVP2z8/brWxWbNBKHdnbYW1Zl/x+p+2Z6DNLXFsLqbbpDvyvusTibwc7xtSI7HZRr9Xz0NNkyfLXFwD9wN5OphINdlX3W9+/q1EuyKgwSpbHiI5cVvOBc+FOPO7XtNY2PVCOADi66o5p7+GZLEosts19hGsH4/SdvbKmyj5DGRMLGVYKsL5JXSN0QH70rMgasw1N2xV98SVVjsAvFedb9J8V5mzBaI7Ooe7ZOQeDqGzoPCNS3rATBd1/p3Pk1m/tZ8zHtPno3WbbcHhZGVC5jxxheviuf2iW8FN2vDva+eRn6OO2dFtvPQeWOO4Exw9P0UUWtZlsD22Lg3NEqCc61La2j8wUrrpGudzFznkWS6lqbz+EmD+8HTnyTq+cyOxU06JMO4doDynsygc99zWaSspEmsguUnYdex6oxCbsmiQiwzpmtL8DWPly+siQK8UjHnMHasRTaYzRrFKxvCEdmJqqDC/j2ZQt6OjP0y7oqOmPu6A91ycQLPTWUAGqVXzjxMKPxs2NordStJcqE1ujcktMUUu4I3M/wrKgXr3w/33mUXjh/8PnNYXc/phTFc7O89s5YfTcbaYbPAePa2fU2mA7uR+IZk0qJhZUqZG463Dfk+yrq/gfJH3QPTsBknVf4kXnGAJXCsLaMumVCiwxGftduLi9ZbuPkEGsun/6Bx0maI0P/GTliqpp/BFWZ/cZT0/PYPTjM3QG64dRo8pM1CxlhM5nVPnhn3QnC3NPc16aNHTcIWTnwO2iT3SnU/Tek2Z/HuuVvH9rlPBpo2v2Z9hbw24SyZTLf1wgQZfSMHeA5QrrkQlQmkzdVqhzlG7x8eGC9qiTTYAaJLj0eKxunF7X34QTUjRbTlFRsdvfqJl6+HF00LKVJkzrKrrSCZAhWSqdt+5hMRTAkCqV1Ac6OJSu9Lywi6NrCE7vk06TZEg0ncF9FPnpNaR27n+MOtLzOCTvLz334DguQrXm2Trli94PqXpHdhCZPLOsh6vobRp1KsDshnqLOlFzg2/acSXdBwlk609IQ7xOKnR5/eYf767QlX2n0G9iZPpKi22iSupjsP24kWFsQQyRFSU3+ign8t2EcNoeZKGhc02/zqZFGKSB+hGErRTco+VSxQZNIU+g5Do8mq4go0YD4GywqSab8NnFco05yx0jBpDoC8LJulrvE4RAsRu61X2xHYnz6wTSyLBXxpQ6YzCDNgloOMoUBCH4EdwmthR15YtUzGwP3CgiiyJpn7g74u3w8A6hcAn+hinK+5ZmbBfLhmORaX2qgbd2ZSfDf/e7rWu0gti6UuOslGyKtOoQwg4DBBgAUmFrAMhKVliIQeOM1O2m/KqAyEjMdqK2zc3D4mce/v72zXv/7r3oLd88KEaqvu8/es82pm+yteRVKgK8qec4Cz/nppmMXY/zrQQzGj11SOhn0K0DCnvribo98AiQDu6GV4mk2VuP6yfBjE8XmO0WHaypgkyBRcURkYLQ0lhD+dqd4Uh7hc0mpfR1hLcGez1C2yJaSmWQtPT99d/fhFJwg2SPzXdSLadPsOwXGOy4WOfYNTsJNor5+8VvV5dX6B2+LZjIm7He4WO1e5s8DXNniOLItvw2Brvbt61GfQqXLEZPz3ZVjtliuoLNUxfh11tOrnbsOMu8VL489116PRZ7MeTTHcqJewXUOy7+29cNN4U5Ih9qkrFvN/hLrAl9ouxGP64arPgmqFu44t7nSFeBFHWs0V+1UVIs/zbnmNxwpg3N//rC/+158ykTC0rCHy2YohvMg4oMnvPObxAWOdISjbClokumjdpay35KYVFis/LN+hscUB+HAZLglJoKTVcI7eq1iFSdLuSNPtlgToXp5KTUePuBjLNmmtqsd/nHcR/DO6cLXHGTwZ14jRaY75Qi72xpN4P/fSc5op4U2Y6Mb8vWjMKLBSMwSGBOqUByDn0jOg29mnPR+B6b6V/sA1sZ3vrGZWyxFonVyUKnbpM0IlEU3qCCao2Xvi8RkVZ+wwCzkCL5Vi7ROSUyHwn7eFjRfVSu53PEBKYewlNKIyjCtC+aXCAmtMHC1GiEbXzDjnrE8+E7FVTF4R4ya90aV+fUjidAK2vbwoTd35kRVOv69A9PQRB0TVW3QUWJlaboHTUYNHVfc9ss9fStXOoXVy6p9tkA/LlPB2vVCow+UCcsHIeLDpojnWToOokL52HR5kIv0yrP/ozf+Xt+ef6DD7i4tm+tdQ09AW4xMYjLpTuvYV8b2B1MsvbcAt/Tu3OH7O/9wc5GOWMA+ihOCXHGAPI4p4weyXraM3n5/85k/5nYVdMcyMOur5z/kQV7XT0a7NapQqUPQ03RlFmxDydbqvv/MMzA9ktXcP8w5HCVM5NBP+rHiN6u4fSIEFtFnKgbFTEmjkMsrcZUS47Hy2k5PWpYbFqyLSjNUxeBjIctum0TXSNJmg/0kIGS8DAroqeHDKAfsCLGqTh9nXl/MG6QfI5cg21GIh8KUPDe5CNTqNU+OtCo0arZv//TdteoPZOC2McBG/nYLdsRcQNN6hKKwy51z+wyLvmlc5/fyqUf6+qrGKCXnDVBFPWCarD1BbulOdIUJu3u/Hh3DT1usNSHMID9YIOlOYQB6HsdytATGN+/dBxjDvZ1D5rcjwYRWyzs4ctf67xSz5G8z5GaiqbzMJdLHWKbjg/p66EvO4bBBj8aJezl1fqnth/gyHXvE3eweyO/VuKuX6Um76v/e8mbuPbJ07gvF5wjrestyxFGS7amonGSfb2KgCXRcf6LtBZI/hiVv68jojHq0JDlNlP0S4Kz7gYP4YBh376Z34XvKXYFF+m592Yb7CqsCR5KkDmtk0c/XQrzwyskFfqFS2x+fLmb5kWkWLBlpcbzW9p9H6PufsX7hjDoYy2bBMt4gp4ZY9kxdTXR1+5gkGqDVZ5Mqds/qd4pJJ939D2MFOV4mJrmWqv6R9Sj7ZthAqfqtsuHVGzJBOb1b3a1lQN0SKV/7UmMuLz6/CpAAhTsJosikKDBaEjlGK9Py6hDxfHY12dFcZ6wvH7HtIOl0OX5Q6KkDt9usBTAHBcrfdRONk6y5H423OTgtooWXBRrupxJzqFv6tcogC31TpBzY3mOaUQc6erxcB1F9a0cjrMYJ/QjtPgKMn8sqmohtakL9+bbwaE1k7gsQM2Kkm/9OdkvQzIzxWSFNMspevo9MitVoZc///wMbbAfJVSvsocSj0J5vQMl/FydZKQgXw1XuKEqtU+h6btqr7IOQkBP8VyuaYcYLFyiU4s3bRTFxej9IV8N25yYVDRnRzVNOESob0KaY+NYYAvETN33B0T6C9cmtEZ6OM7qnwjqRbZUoZfoQhBc6orjplnZveR6CPoDgx+B3MrQKj++RP9mt/sc/fgj+jdEpLL6sus5UA9T+x/c/C/7RabRLlHC7S+EzOmjtXXFhmYEcz7H5CZ96VNOhTT1aDSwKywR65oXME3GptIBcyRvZgQsAw23MQeM3Rx7I5XVrMXWaR32g04zihBSCC1kJXL7wnAYyKChI8Ddkhd3b8QAcoxYoL8Oe8JGI6ew5RLnj+Wd8+ggzf6EYZSKkYDV4U3h7pfBFnbPfS2E7bOPTavRykV9bDP0q9zYoxnanEwgqawxZiS6obQ8QLRH8eJ9JURzgymydcqB5xe15IGxVG4+tYBJ/B27cM0UjEy9PN/1vYuAi6M70x2I4Xbhr/rlOVJWWmtwqAxni4xO/28okaye+eSU2J1HMpIvlyQUNBT8bfOrD9ANv5nRTBTFfhDQiKC0/6sDMV9B4MWvlOmSs9TdSx6tOa9ZqkLYB6ZIH9c06q78DrfOvgH1RCDPdbXV4p+Q/x4RRideBuOCJonRwwggqdDV2Zsrr/sSLCx5WFFK1dd4ETyRX10aRPU43B+f3FMFhnho1C0amvJV+5PWYHd6DljmM/Ty51doA3QvKBYIcx72FdTVzwvU+o/QhirqwGKDOMXaICl65SK7RDy5mvh1EzFwV1OEbT3tfpcqB8JBVhMlKyG5XG77gbgFUwMtFqGfEVlhhYlxRKTQvshi4Sa4o0r4nB6+4zMfraiNXdDtAvUpgwj7pi1Yi6KwSqYUdRhB4c2oTAPJ2lMrMQGN1cUohPc5SEIqVUPUBoscqxwJqQrM2Z+h/F6piiB9cp/lcDSJ7jYLbw+RWqwbZF5wtqCw44CBrymRIh9RsNvjzrSZoKF9aENMEFmUnJogA4w6UTEo8OONprXBypyIka/t2kF2HmPlXc4cZb9CiuidkPNBgsSDmx6I/ESEvxB5CrJbkH9KcaLuOfXqtYrp0ms/9ik8EFHJbvQbBMO4/Qhy3w63xi7flwcWON+HMtu2Pwr84SAVJVLlNE/3DvokG/9M6WbFWseoM22aL3bj68PXSsliBlArKMrXhAqsmHRqfVFxw74zjCqEy5LX1S9tL5sCC7wMleYixCG8U9uLDimHq0bMPNFIboSLjBlclH3PoMe4npo0vH1GI7Ji1rqROdUz9K7SBsykLlDXPWskLxcbeuQh7RVgi4XFe02n0ITgkOsFHe3c0DRBHENgq1rnbM1yq9kAP4QF2XUtyD72iBfe5G3J1GQ7bM/TxYJuLScyw7dus9oKPauvWaSAQff7RiMe+oFu37U8mw2WbLurVbElUBF9FGdD/9hXBTTILxWtJmMly92Oi1r5uMEw9rTqNuDqolkCcrFGPTREjagU7BA0gUxbFibB67ssUuBaZglQLbMU2nMZUxTtAo016qOFmkBX6rwipzEhe+Zj8I0ZPJf3enOOFZuH5NoxwYL2geh1Q4jtCMJkoMTHUKx1xU/UNF9WhsiCvnA4NMaLH+Ay4BAsPAl2DMgRBqFrqphJ3Rp0rPu0X90XAY6NJu25fCYe3OZe6abSxUKDuJMbdd8aPmHt1gVzxnqqeF05fTZT4AAaFyPLB5Nhm0mwQbxDU2QSHsLnXSu9awlKhX679qmxTNcJAX2/Gqxfn9BYlaQupWYRBcedeAvMaZG33YWbuzvahafiJkvXuuieokhUBVWM3FcWBfc20eTnO1SyNTfDiSV3vwdbW1ORw5zkg3JLzv84QfeaOrQrh9Npu4ilrwUfkBvmAe9FzEn6lL3qvhmdBOvFjPdyrXCTWyykQbiZpBZOoOVymdWJKicR6jUj3luoT9EzZUf2/R3SraBr9bDtd6P4S87IdoppOyNy4QoQ8M21Bd+OyOWKp8ybDhPwQ+Wb/4fFqRSG3qbWWBuELttRAXV1VZ5r+3/wqGJeIxRqAHPgcSYrLJY0E3STWhaMBS7pphPqByXEGMXmlaEdCTHM0dcOdautd5+/kaHEJY4m7BrK8cGEjkluDhiC/fwih0xXfwsYt1ABZglWNxzUbc6XWlM1Q9fUHUqlqZrhJYVW3j7TfSFVjcMAdg3G6e0Efo/c7zt9K6RCcyU39rP6r6Se42jNrtF+0pf5FVYmtpuuARzbo+LvlBxUh051pyTP2xmkia6ULKkPKKZ6i98IhDlVpskuUu2i/m8uvOXFR6cJACQhBRTmHAkpvlO0pGDJ7Mt+mGIuym4f/dA0FKfHvWAuwlaHfwY780M1WlmPzmHBOVSbCCTFd0tp/3vPSwBKShZQHBPuG3eCgS8AAYukXCCYMM+onqHrVqb0Bxt0K6vSYHzmyvkqbY0YVzLqkm1yL36baSaEV9rUDOn/MTgm+AnT9iR9TbT3b1jFFz4dV4Em137cDQtb9K4tUzql7Mkhw8tieQ5YIKy1JAz8pfY0gvYkHNhbdkNfdwYZwuDC56hUMBPlOaKGPAkryljhWAOrDwSxYClqqNKoxBq6eGlo5OCnScuisFJM7gTth6U11JC96p57D06l8XXOMMHD5MQ3kUVZDe9ggmPDaMNELjc+n9ZPm3zeZFKMEmOwzUXF+RZ9qTB3zs9cFpj5Qbyw73ohLkeerq7XM9EA+8FoOCZuaO5rgepEdKzBO+UNFPvJNw1qM5bvOzg+6AqRVNR1Jzs5t0QfgRq9365Phddvpfe8outhu54m6ExVwfqDnVK7WP2anTF5+zXtHyNr2gvG09/xZsu/wGrNNVY0rwhFdeSIht1tbqZ+FnhNkz0i1ztj/PvvY+cBtC/MqF+Akht9VMuBGB5jv7p96FZYr5obatXCQJVhRVYu87eusWnKDM9qSL0WYXYjzTIzrYj9VfPvYaUpsvJcIAY5d5UgnGJl/wSN8FrUfAFhPfm1Luw8HH1wwq8a9nl61C8WkcW8Gd+72HmwfNmousfrtWaq0lN7+rraCCAw7vGbJkAauBJnbnXXk3HcU+osuOkG1zov8+W5H8GNnvrGDfVsSlf0a3F7FtarnQP6VAP+vfv58rw737URE0PvwW5EzqUBui3MHBNZWbBhOmykrvU2ZS/73aiuL9B26sJeP7ZwxvfE447PmoXR5flBTTaWf+6AJmsReynyVqOdoTNXn+n7nXL3wX5tFhBUu9/44RvvjptXpqnclKZ5jCrBqXaUke5B2Ui0xorhOR9UAbqmDEygkuMRQaCp0En7o+wcaFdVdSvPrKSyGkZdX8jsOV+/uLzq69DIt4x1HoWxuuwjBwreuRayjbQ4JNGlMOiaLQUGYTHCoqVUKZvXPhnIL8ukV7XuJqGrI/ynRaRzl4HLchlgnPe/fURMEF7l1IozP8jW/nyGnl7c4qLk9DW6cg4RBxak9yzsF4HI3OSxTXBOtU9LGDOmb6zKfQRe9yjF67gx3/un4QPTN3tCrkax5ZKqdCPswiT73I0FeBxAO10pqleS55Z7nK0+Mml0J/Q+gWdhGHv3UvnpB6djPGuacVyeh8tI7hydJ7Ios4nzruBUfO4VjHF1/j1dzb+z6EgB9akLGDcj84qMWWleLT1R1lgX80ZaSgWdB6xcr/EbmRKHVb7B6jQZesOu+la6Yv8Q2U2MtEZ+aoUoRu8wqfsph5VbK4ImtWOk+K5WUNV+KeRszehDrRXFOnpusDbYVLEU58YfhRk/mdlhF5/LW8TyF+Pvl31ZqykwtBh9GjQ+dnfBYhG+uvU7lnj63oDJz4dz9455zpiQVawYZ6eORC+j3ykrSWM6HQYe2Z8iA07dmXGHJd5wbuUe0hUhVOtFxdGFXR8RmVNtWaJu9hu2LJjI6W1kAnCmzXGa5wNlCywMppiqkZhTBfHNAivGIYMn4MFz8XexRBiI+J39bXBnIgEfyrlrLnQijdivjp42+ZwlVbr0RbdOwgxI5lWENiG+7vD0bKTI0Lm5hu9x6oQSp3w1SV7eV+W+bT/ETGiUU4MZDzgZ5rIynd+NbE3yyXMza48tbvLYAI/xh9TQouTJsnneoJwusA8B+c6XdQzfZ2tarXhNFcdbKOQy0j+u6GngRtoPwOr2v6aLugrc+eq1YaaCxowouLHWNhg2bHrodY0axer4dwiOjWkCWUVkUdj7lIaNzhx0xDrJvqWSa5Y7/1ndRa6gejQRKpfk+EDj/b1lvzDeao2km5cXVg1uS0h6Oo2sr1dPK+v/kPMj/U5Hb+8/5NwHYMK3q2TpGueeQ0KxO/nrq0t0OVCoumgk61rrq0v2YxCxsKuphl1GNaTv4w/zudVh5d6JiGwu89QVX4OKu77S4XFBFpcR9WgVv1uCCxlMUHnecQH70mGXQNvEQ9iS5U0oZ8SJV8S2Ggdl4BFe/nhKXrPvskr5TNXTva8+ue45dSAKkjVuKam6XgSX+jWnofLWugvTvsSNCRwhQa94vusQaaor8RozjoeBDNS4whHUVy6oUiOTFtwdOsbXHy/u5o2VwjeAcgHYwZZ8uoFmy9mIRGRFNq/yfBvdP8OKLGodUAdupelxjc73eqniQ1RMRuxy0Cuxy3Q1RUEC093sVddzFVc5M01lXdsXzWMUGmzXVmw4UdKGF/Zv0mWJxabgejKr/OzzBXrqayU+V9zqynPGoYAD8sAubkup7Tefoe+GjgbRj8LcCLkRO4aQpqSCZhbrXegjkzYJnsAF108LPaur3N/70qS3dInJFn0aNdc4myt8iqJ8v/AOiZlABWZioXBB96ZjlFjB1N70fRJ2lMsrWBa9l7lLjm7bAnayzgJIoQPaF6QKWEKkspB2+8a9pxv0ayXAlHwnc8rRUybWs2+fIybJczS3/0ft/2GB+VYzPfs2HF80pMwWHA8m58fWoXY1/LMrBIuCrwvk5LYefiUXexs1GJkUU/fXucezboOgqbKMHERoXcSVuz3MPr/7HSuKProE4G+//fzu9zcfLr791uXcrrHCbJQnN1LdxCxZPnjBfq8X7EbYRp1gWMRWInzNTtwuJc1zgIl9LrYJTJiFVFRoRmIKkI4rKQHGRXwvSCA+EAtotsFsOJz4wd4B6H0eG6i9PrFL1HU1T3QpzDzXRsWufId67WQOse5bGu0drWs+0jlJjy12aQeDDVQaX2zS1r34ehcLYsFGHU31VpM5Yo/darAbUWCb/fKesFA+up/g/R0XFnmv/38YrtqqzG7y30lYLO/46D0ie5E8CXPUcdx9+Ek5QdLWzsl27NKnpslor7PsoE/mM3C7DTj3cGS6blnNpoiHQdHXAjNuaV03c7nyMuPyvFvbBp24rDlo6DLQwmA8q7DOuc6sinjEfo5JvIZ0a199dCaLohJ9T9QAO3Fc46aHYvee3pq/07BO3eCmj9OsH4rbNRb5v8tw1KzFzWDDjpEMD8ZuuPAOcrrSJSNMRssSncqCB+w3WIlh0OGxo65FUWYylTC+fv/uCv3m/KhtUmoYkS+TphJc/+db9KWiaqR3a8VFpmi/U2fa5IaOQ3SLPtRFZ8G0rkZLJxEf0i5QGXuMgAVaHuU4OgTVBIJjD4abxx/QgDlWRYLTsmATuBdwGbEAuQFa5dGm0u7AjNvtagd0jk1fK3wo3DkVZFVgFauspIG7LfFgfPGDo0+YDNKposDMVtF5gdBF3AKqBvBiCa2WEoCV8z8SQC1x9EkYruNUdPaCoHvGYj84vnNbQa3qGR1pkWECg1Hil59Y2FpENN47gOfLcv2TuDWr6O87ERkxKst11L7rHegW8nGRpzsAXnMcXWKIjIolExGLIoegU+RGi2yR6Q0zJLr8ENmCy43GRfzclS5sYdbpoCeIuhCRMZFSnDBRUlXMt9ES3gewS3KTBvga8xS8wsqsVNLILH5ICqCvf8rA4xgfNk92N7lcZnkKYlvA8fPfiMgKfJsZE8ttsAvYcjSnCR6FgolESDORDumS64zPeRY7LLoD+/uEwKN3Bu/Ajt0LsQs7dlVvF/bPCWG/Sgj7XxPC/p8JYf8lDWwjS47nNIVIaaDHN89EVlQclO/5NsE7WQMvbxLoJUXF2bIo02jfVsvEfBk7CclDZimUEk2/kPi+EZFpl5CY4AS1ImmsSQs4jTWpt7oqE8wiJaIpq05iqhpprOlBbxOIECONNcxSwQazJgnwSrBbgYXUlCRgwvUrS5VEj8L6lSzNiuI8gVtNFmVGeAIftgWcIEgCcNV8a+K7RS1knQRyWWUJYhpEMcMI5gkKiHSGl1SQbcSsqy5sgfn2T5rPU+C9zqANaBLIrh1MGqxdYm0S6PNluX6Vxgetszkzf0nSaIzoLO6suB5gJaOLap3kmgNUSlT8KjftfPzRZm11AFOzcn7++M4RBxzUviTAXTf5eB3kOrAXjNMUNozOFikOkS1iFmfvAk6hG+iMlZCkmCURdaxc/5RrUw6a+UeCrRVJApuzBU1hxmhwNBc0Z9EKRndhM5GGSwqZV5xqIlNQ2wNnywSySZZ6g03Umf8d6KEM8iiAFV0ybRSO7wlpYSfQ+BQtU5FaJaO1hk7kKpF8dZn5jsUTQDeK4iKBIulKgVKhnU653qwk05mbMBsf+hYrnITB85FC2BiQ126+fWy4TBssos85zrWZVyrWsMAaKnWzglJAraLjGl+PrmuSY4OFyQ2L+MOuj+00sA/mEud57DvA8thh1bp1UIK3iBUZUVIWSboSWcAJzDRWZGmSI33HoxRkLm+it2cqdfyWpazUpWKRgXJsmKmiZ59xJmi8FjstVB11ok4DF4pv47u1uHRdT7MFl9Gf8wZ4gpR/a/NGlzoWaAKJY23oBKhGz03gcpmEdcUyyQUupYotwIp5tUxxzQqmSQqxUOgkDJtiDoSgBporRYcbXYa7BtCxM/4c1NjpeGKziW2BJKkok24AdHRLVMbXjKRiyywwj+vBcDeCqvhvVpm5obzRwUadTN2CdSNekzBZgsJNPxMntjDwYGNLgzJzjqTo6GKt7YcZWcWq8x+Aprclix4IKKkqlgoLM+i5GwPyJgng+E+v60T26VNvCmgEwEouM6zLiAMDuqAVjg1VUcxT6HeKEqCD6zqaCHh8IlvIcVu4diBLlSfAOL4jUyfwDWvnG06QD6Bp7EQAN/A4gXGi6Zf4DBBq0BoNagJTSrNlAsGry9heNq1IinugSB5dkdaKhLriRgBs4o3Y6sKsdPSummsiYhdKBKfFPhSoa9IZe/tmaeKzlQMaP6LXzPSMDXdbRu/WWuXzJHnoleIJ3sJKU5XlLHbVe5KxFXVkKAUZDNEGF7G9weuMCW3wIoFmsGbKpFDD16VI0LrJSFWJmG7WUFu0QEfRN5WR6EMl0GDpJnsk4bC8z5izHJ0pmjODzrDKfTdDDe3fw+i4yVkJqTQ2IRTAwBB9BP0NiOQoVKrT5EMwkY5yF0XJ5ZYOBgsepN9CVtGaet+RxywNnc8I5p0puqS3qMD9RgttLFYsq/4wkORIcqZhOEO9uj96aKCEdFWWUhk0bDyK0GaFDWIGlYouxljhAWm59xlCESK8tzoaFBATvrP7SF9ozkTqifwdVO1qXTw1MnJJzYqqWft9vZLV4EVDSNA1Vc04IiNRiZWm6B01GCaCu7uKGxI8fSuX+sWVK3t9hs79iK/nyKwCU4qgGfAH6kcfA9oCvafmd2YE1eFzHjJ1EuItYGR3c4tgcbdZTbEiqxkTLIgfzNydoL92T3zCLAxIhnjBcSVg1u+ygjmudRP3cAP3Xr/2PXtK34672VPThNvPLx4x9u1BZBFrmu7WeRWWRR/prYFbMeYumGIa9YhAagfXvYcJ1YKPTLyE7rkJx4FD/1xNDVL0S0W12dO0+/hs5fv3yncqA4zlcas6id33SDV5p7vulH04OYwgNrbzd+jQrl8Hdx5z9v/h+YZ2scvzWijA2mHeAKshXhLvPVnYPi5zrCly6doNNmhwq5pT8r84Db6iGQXfYC6Va18fJCNCWCNNKYw7w/vnVSksNCYTjPcddJh2SwtQe1umIZWCCWj7kC6pKphTN6ZCul3SDeZga8bpkiJO15QjrDVbCndw7bz+MOtDS+YTym9Yfw+nz08y6dliVgn2paL9MYk4fPk6+B7XMfG4KSi1RsNydyGJFIJCbgXaMLMaExQIBSpDGo1d0aPKi+5tWlhygjxpnigul4xgjiwGI6YPYHFa7GCpkTGNp6NdudrqMHqddLaN7GW1xn7gMWdYZyuZ3CZwRlxjrsEslXaokZWK3RE84X4AyF0aiy28aX4QC+EUq9kbrqU1xHfu2zkEy9Gv/hcz9EZsm38NoBuw5bUwCOczIouyMlSFxXASN77dWDrz7Jv+WcCMxZ0DYeaf1cvvf/iLtX3PO8dRU+ybINqeT7O4EbO7Om7wlir0r41PTr/waABy4Vsfu/4nPc+LFucdrt97HkcmLx+SbU/6A1PsOjP0/rePF3bvVFHnPAF/ac40UbTEgmytVunVM97PBUFAoefo47vX6FKYH18+R5fvzy/+6zX6dCnMq5/Q081qiwRlZkUVIiup/ag0qRQlBr71w6v//f89exKkCDWrhDKuTw+QqbMCh8fx6MTcd89rfu148bJGKnzF88eFdFc2HcD8yIZxd37gQ/j2FNPWOvnMlKkwR2/fvA8i+6cUNJ0v6zjO+D9S0FmYthbdr0aEwkYOC084gsf4Bu85hyU2dINPMCIduPsKvclzBX5ax+UhdJqnlxTlsXHOh8ZCLs/eXblXaTQ8VmA9YfRjx6nkNFX/dqPLK4vKiPfL0vDISRBRaGjXHqdhrYllbrrWtAKigy7Oc2a/jHkbsO3M8g+/cxMygDUJ4YJLf8PPd1lggEqba51Er7vrk4bRe4/hlVSmEckDoZtDgA0OgJntYcmrJ6a92w8Ty/oxqbf1bozwgobsxqm8uB47sHyx1pIwq3I6v9FAx0FWLisslnTWmE5EigVbVormaL4FmFTkkDUUljPlka0HBkWjI9pycNFFgn4HPKLu3y3hiu4AULSQhmY+szt+nlF80uZCZzhzqfgJQJdGpQG+SMASiwTVwjzFdUjV/6RMQFScZ7UnLp1a3rfg7T5m/dW6zoQTaLAXZkWVoAZ93Jb0OfpUP2NvwQH2I7qqHWCDl+C3MU2tHtUzgTIxYhrXSHu/+HOEOQ8qE2X7RUhwwwoS89ZU2TeQCSORNvCYM4E+XY4KFAIJssnkVXSRbYHKMsHYNwtYUR07o9eCTVDi4l7E2Kno4G9PgK0brZBxKpbRJ0UCzlb5SKiFjmigTuXBvBOAEYhAOsECYfSLVBus8uGcboTeLCHZSyFsb/wt5NLNqdlQKsKqZ+SuifeNcUuDeTdU55BB0DIeMiMGO2TC57lCWkLBjBVLfsRGeItrjsUUcfw7OCjrBJGOi3KwwV2XZRtJWVsLdgkG7O7LEztSSQl0IVjH6wd3t4g9VoaRimOFoF80qpF4enH7+q1cysUiPP2dksysaPLj3UH2o13Q3cYO3hcWb4vum8qsqDA+WXwUbV3F7Jxwt4Qet+Q46p80VaMIy8oQOS2l/ZLjCF9XhFCtR3CGzuPHNUc7LvEE8EJWxV1KtUWBwoQBblMIpx0caQ9HK5UgwKdLKey7YuVWSDlsfogGitLurtbx+tGNvJsYua6lUDPAGc2b/Xg/TE8fZgJpZqqA/ERQXEC9iPZQV1gjnMvSvi5mRZlCciPaI3OEM/hWClmM5NXCTA7NXIv6aZUIq9wzkVv5I5VuCIDRL4xT9MYjNhuQ4S7OXtFszN3J0YTxZv8nSVcYJcG1z1qIS4XQHgOEiFnv/gBCuHy9a1+vEZsS4wmhc5myeiCw+Tld4TWTFWiXRBalkgUbyVCkUyN3IfCcQxHZAp3tx42JdSN2EiLZx3BH60RBBHYwjDpc5ggEA+s3+KU+3c4r2963UbZryywrYfrlbLE1+hzKwDNyjFl/Jy0I3uMlFVQxUm8JCAKJfv3UAmZW8NSGZrshj+yM/DDTRo0HP+s9HdN262R7erl/T169cGsl3FfQNG2McMMKqq1cd9qeoiUdDSL5U4jWFOLgQUDjwQceg7ojax3Tu/tkrPXj3fb0Q6ajDTm989a8w/jQDgd7gx23AuEOwuDr3d3Lg7tTk56du2hR9qYOn1y0XqrTCJADcrwRIF8vO/54+MhijTaY5sjuJh/VpBIk5h27g/yYlB1j7m3AjI1SDyVoPT919MqdyqyygpqVPEGUBO94kpFDw39t9MChl5KSSb1Oe6I6HyT3/lqLyB6+TOQJ+a/Zz99/j56+PX9z9QydM22YWFZMr2gOpfBBXLhcyuR9gfZFwiBbduHw8McMXxzJGFMysVdxX/2nPdUQBs2NAY98tKHP97kuBNL+m7rfjuMPcArFTLEItUlvM8Uwj9WdrreRDzhnlXYrIKmQZgXjWDnxZMWmvUME3vVweRXcc83yKTuNdDPlP1lGqL2Ivb6Y7SVPV2fxRuy76xDW8JWGHf+vdxLBJwNe8I4b2inLyMOuTKlSJgYMQjZAaqmWWLA/92RVi3SscFdiH0HpLk+NkHvBVLCWNFHXn1/scvBauBZfrnfRTlbzrxRzsyJYUVQqmsuCCRwsuOuIpytsGBVGH0yP53jK3b7FJ92sa/1Iy0SMa6/OEyu4SqwMNENqt7pfrE7Y7MgLm7tI1AXNqcKG5lm0pLI9/GGFzy/1ik3w7ErJNcub5mH+e7gsuddUB4zhm//YZ21Xpw0rOO0mWT7RLpslfa8/sx3ZZnB4KGROrpmLnq/6ivtIC7hG6Yw5FPy+mie9BZ2p86NOJfQysFGno4LGijXSRion8S20ghoMqz2Bb83st56Ed1+wPOd0Oin3Dta7q5wLHG9H7h0l5+rxGNNs98qv1ukwJLZ1dPY5Kjm2R2bfZ6kQFURtyzEvP6RCTmBP3iGDTjW25a9SG/QOkxUTIyZdjhNJjm/6tP4kINO/VNSKD6sfuSZneobe5rhEn+EfTj/KpXB1p/8cPp5ohdfUak6cYoW+VFRtEfQg1KUUmtYaVbg41e43g99MIy99DzxiIStWd4EUbvuuL984nvWWJkC1ZaAPvjnqXTGFKU9pHWZ9Hq9bS+80MbK2oX94mUaqEiJox+rnzcvjIs+ujdRIjZ2HmHkLM/1BYLRhIpcbjXRJCVswYj95HqoT9Hmywwtit+fwbXNu0FPoCEsFaZ8hCF0+61ALVQLe8bd0ickWfdK7jW+bCGzRL6SNnl1rV5jAYB957bumFqACtWrAZPZFHFC86QMQqP7fqTSFcp4h+Xa3nV6hHuvO69TrwI5hh0FG8785YrPT5PWObdVn+HrXey3rLmDr411Ah7uZxmHXBAx2z6ZNyHTHMDihcEOKw8XPUDYQcyTgaIUbbDmnCya8rx6EE3T1K3A50nQQsDuqUCwRbq0Dpqf+xRaMjc829d59L6WR3pSND9sYTFbFxC3w21WB4GhgHXWPI8mQlzkT8SaIRb0bdstQVJj28QwIqW7ZDhyLa6PdlvcHpnYOsE779h3AusSq5in75+ftVjYrNmiljuztsLasS36/0/ZM9Jklrq2FVNt0B/5XXWLxt4MdY2pEdruo1+p56GmyZPnrC4B+YG8nU4kGu6r7re/f1SgXZFQYJctjREcuq/nAuXAnHvdrWmubHihHABxddce09/BMFiUW2+Y+wrWDcfrOXllTZZ+hjImFDCsFWN+krhE6ID96VmSN2Yam7Yq++JIqR+CXivMt+s8Kc7ZgNEfnUPfsnINBVDZ0nhEpb9iJgu6/0zly67f2M+Zj2nz0brNtOLysDKjcR44wPXzXPzRL+Ck73h3tfPIz9HFbuq23ngNLHHeC44en6CKL2ky2h7bFwTki1BMdalvbR2YKV12jXO5i5zyLpVS1tx9CzB/ejhx5p1dOZHaqaVGmnUO0hxR25YOe+xpNJWUiTWQXKbuOPQ9UYhN2TRKRYR0z2t8BrHw5fWTIleIRj7kDNeKpNMZoVqlY3pAOTE1VhpfxbMoWdPTnaRd01PTHXdCe6xMIFnprqADVKr5xYuFH4+ZG0Vsp2kuVia1RuSWmqCXckbkfYVlQr174/z7zKLzw/+HzmkJuf8ypCmfn+e2cMHruNtMNnoPHtTNqbbCd3A9EsyYVEwuq1EjcdbjvSfbVVfwPkj7onp0Aybov8aJzDIErBWFtmfRKBZaYjP0uXNzest1HyCBW3T/9gw4TtMYHfrJyRdU0/girs/uMp6dnMPrxGTqD9cOoUWUmapYyQuczqvzwT7qThbmnOS9NGjruELJz4HbRJ7rTKXrvSbM/j/VK3r81Svi00TX7M+ytYTeJZMrlPy6QoEtpmDvAcoX1yAQoTaZuK9Q5Srf4+HBBe9TJJkANElx6PFY3Tq/rb8IJKZotp6io2O1v1Ew9/Dg6aNlKE6Z1FV3pBMiQLJXOW/ewGApgSJVK6gMdHEpXel7YxdE1BKf3SadJMiSazuA+ivz0GlI79z9GHel5HJL3l557cBwXoVrzbJ3yRe+HVL0jO4hMnlnWw1X0No06FWB2Q71Fnai5wTftuJLugwSy9SekIV4nFbq8fvOPd1foyr5T6DcxMn2lxTZRJfUx2H7cyDC2IIbIipIbfZQT+W5COG0PstDQuaZfZ9MiDNJA/QjCVgru0XKpYoOmkCdQch0eTVeQUaMBcDbYVJNN+Oxiucac5Y4RA0j0BeFkXa33CUKg2A3d6r7YjsT5dQJpZNgrY0qdMZhBmwQ0HGUKghD8CG4TW4q68kUqZrYHbhSRRZG0T9wd8XZ4eIdQuAR/wxTlfUsztotlw7HItD7VwFu7spPhv/vd1jVaQWxdqXFWSjZFWnUIYYcBAgwAqbA1AGQlKyzEoHFG6nZTflVAZCRmO1Hb5uZh8TMPf3/75r1/9170lm8eFCNV3/cfvWcb0zfZWvIqFQHe1HOchZ9z00zGrsf5VoIZjZ46JPQz6NYBhb31RN0eeARIB3fDq0TS7K3H9ZNgxqcLzHaLDtZUQabAouKISEFoaayhfO3OcKS9wmaTUvo6wluDvR6hbREtpTJIWvr++u9vQim4QbLH5jupltMnWPYLDHZcrHPsmp0EG8X8/eK3q8sr9A7fFkzkzVjv8LHavU2ehrkzRHFkW34bg93t21ajPoVLFqOnZ7sqx2wxXcHmqYvw6y0nVzt2nGVeKl+e+y69Hou9GPLpDuXEvQLqHRf/7euGm8IckQ81ydi3G/wl1oQ+UXajH1cNVnwT1C1cce9zpKtAijrW6K/aKCmWf5tzTG4404bmf33h//a8+ZSJBSXhjxZM0Q3mQUUGz3nnNwiLHGmJRthS0SXTRm2tZT+lsCixWflm/Q0OqI/DAElwSk2FpiuEdvVaRKpOF/JGn2wwp8Ko7b/8/wEAAP//sMlYiA==" + return "eJzsvW2TGzeSIPx9fwUefzhJDrlly7b2Rjc7F9ru9rh3JLlXLckbFxNRAaJAEm4UUAJQZNO//gkkUMV6QZFsEqhu7d18mLCaZCKRSCTyPb9Dt3TzGv1RCVZS9S8IGWY4fY3+w/0B/cen97/d/AtCOdVEsdIwKV6jv/0LQqj+DZozynN99i/I/9dr+NT+7zskcEFfI0HNWqrbMyYMVXNM6Jn9e/M1hOSKqrVihr5GRlXtT8ympK8tjmup8tbfczrHFTcZLPkazTHXtPPxAN36f+9xQZGcI7OkNWKoQQytl1RR+MwoPJ8zgpZYoxmlAsmZpmpF87PB/pTG99jMQsmqPHwrfaJulwWsBead7Y2vPrZ+aIntIoVedP6+e4XxAxucyscl0/Z7iGlUaZojIxHBpak8/RVeo4JqjRf239ggIguq7aal/bwHGqG3coEuKJE5sHFgIw4W6yN17HZquHRFhcns1iID9ggnpr4nuQaaEykMFUbb+8GENliYGg0dxNGw4hgEc2z6HwyxYw4nuwTCBq2XjCwRRppqzaRAS2Y0wug9Nb8zI6jW9emfDVij2axeyornSNAVVWhGG74rsdIUvaMGW9QwmitZtJZ6+lYu9ItrTG6p0c8G4C+YosTwzXNkPN4YfaBOWDgOFy00z4KE5HRF+RGU5FL072eHkhe0VJRg4zHJ6ZwJmiMpOKBl8IxTVOAyjFWhF1m0C7PjjN/5e3518QNaYV75G89yKgybM8+d9A4Tg7hcuPNSg4OA3TEL3nMLfM8eR4mVYaTiWMHv/cGejXLGAPRRnBLijAHkcU4ZPZLVtGfy8v+dye4zsaumOZDTrq+c/ZHBRvrH8miwW+FjhF5y1BTVslIk0dt7OtlS3f/TMNMGG1pQYR4jcrjKmckIx707/EjQo8KozWNEbGl1qseIGBPHIZZWY6olx+PltJziY6RHWrLNKc1j2lAjek3Izmx9sXYLWGwGeshASTjNiujpIQPoe6yIcSr2XCsTUVG0vCpB8jlyDbYZiXwoQMF7k49MoVZXgn2p6FaNVs3+/Z82XaP2XApiHwds5GO3bEfEzYqlFYdt6p7bZdicEdy+z2/lAl2uqDDoBoQzqkROlTVBFPWCarD1ObujOdLUWCCdH3fX0OMGS30IA9gnGyzNIQxA3+tQhp7A+P6l4xhzsK970OR+NFhKnUhfbfPlr1KbtojkfY7UVORMLOoPdYhtWj6kr4e+7BgGG/xolLBX16ufEM5zZWXl2HXvE3eweyO/VuKuXqUm76v/e8lrqZVeNvTlgnOktb1lOcJowVZUNE6yr1cRsCQ6zn+R1gLJH6Py93VENEYdGrLcZIp+SXDW7eAhHDDse7YBKl+6pdE1XKTn3pttMPq4KSkieChBZhRRZpZUoU9XwvzwCkmFfuESmx9fohnWwEV1gGzOFpUC1W/Pvo9Rd7/ifUMYNJ3xGcG/YH+9kKncbLus43rlr97BINUaqzyZUteSaK1ttyl5df25o+9hpCjH/SNFSG+0oYV/RD3aFtqSOk7Vjnj231KxBROY17/pait76JBK/9qRGHF1/flVgAQe/QElTidBg9GQyjFeny2jDhXHY1+fJcU5VZPErn+FpdDVxSlRUodvO1gKYI6LlT5qJxsnWXI/G64VrautogUXxZou55JzSoxUX6MAttR7gJwby3NMI+JIR3OLaUdRfSv7agvaQehHaPEVZPZYVNVCakh2K6RAs83g0BBS9EtFtbEANStKvvHnZL9sBT2imCyRZjlFT79HZqkq9PLnn5+hNdZIUyqaVXZQ4lEorwdQQpdSaJqOFOSr4QoiK2Ean0JVzJzQs1dZByGgp3gmV7RFDCaCmZW1eNNGUVyM3h/y1bDNA5OK5qzq62kxCPVNSHNsHAtsjpj5Z/Xy+x/+op1If1GCAK2R/udgN/+09uBbvKEKvUSXguBSV9xFVqxJeS+5HoJ+YvAjkFsZWuXHl+jf7Hafox9/RP+GiFRWX4Zd+EWfo//Bzf+yX2QadYnyTfAIhczpo7V1xZpmBHM+w+Q2rQbskBPSwLXBxtkVlohU5KVkwoBpYmg4wRmYI6NKyUT5aVt9UJeUMMwBY8BUG6msZi02TuuwH6wwZ7ljjBBSCM1lJXL7wnAKyDOx8MrR3uTF7o0YQI4RC/TXYUfYaOQUNlzi/LG8cx4dpNmfFBXUKEYCVoc3hdtfBlvYPfe1ELbPPjZbjVbO62M7Q7/KtT2aoc3JBJLKGmNGoltKyz1EexQv3ldCNCUJ1TpbsTzLU0VdL2vJs6CCKmzgkueWgi27cMWUqTC3RnvH9y4CLg5WMGt2Q6wciOF24a/61QVSVlprcKgA0bBaUNN8bS8ltEqU9PTglHCZcLspoZKEgoaC/+qi9r1+oIU0FN14fieKwkM724wJSvu/OhDzFQRe/EqZLjlLmdnwqM15zQZq/6PQzazMTcjvcOvsG+B5vea62mrxT8h/jwijEy9zxh8gRm9XtcbR9fmba6/7EiwseVhRStXXeBE8kV9dGkT1ONwfn9xTBYY4mO4hV2rXlK+2P9ka7E7PAcv8DL38+RVaA90LigXCnId9BeDUBzVp6z9Ca6qoA4sN4hRrg6TolYt0ifjgauLXTcTAXU0RtvW0+12qHAgHWU2ULIXkcrHpB+LmTA20WIR+RmSJFSbGEdFe6g3gD05zgSrhc3p4x2c+WlEbu6DbBepTBhF2xC7BoiiskilFHUZQeD0q00Cy9tRKTEBjdTEK4X0OkpBK1RC1wSLHKkdCqgJz9mcov1eqIkif3Gc5HE0iWc0GT9K9iLTFukHmBWdzCjsOGPiaEinyEQV7e9yZNin9LDs2xASRRcmpCTLAqBMVgwJvFOuJwVa9mTIPxMg3du0gO4+xcpczR9mvkMIsIx3Ttj41Vs7LNsspfyDCX4o8BdktyD+lSN1tYYdYtKvXKqZLr/3Yp/BARCW70W+QoXfGXz60okq3yinyXXlggfM9ldk2FMfa5rZMj0iV0zzdO+iTbPwzpZsVax2jzrRpvtiOrw9fKyWLM4BaQVG+JlRgxaRT64uKG/adYVQhXJa8rn7Z9rIpsMCLUGkuQhzCO7W96JByuGrEzBON5Fq4yJjBRdn3DHqM7WoWxeHtMxqRJbPWjcypPkPvKm3ATGoDtbcSm5G8XGzokYe0U4DN5xbvFZ1CE4JDrhd0tFN0ThUVxDEEtqp1zlYst5oN8ENYkN3Uguxjj3jhTd6VTE22w+15uljQneVEZvjGbVZboWf1NYsUMOhu32jEQx914Ty30riRZ2eDJZt0MlnFlkDFQJE7FWJD/9hXBTTILxWtJmMly92Oi7bycY01AiTyEb4B5H6ITdSISkGHoAlk2qIwCV7fRZEC1zJLgGqZpdCey5iiqAv0ZXSoCXSl1ivyMCZkz3wMvjGD5/Jeb86xYnOfXDsmWLB9IHrdEGI7gjAZKPExFGtd8dRhpxErSlaGyIK+cDg0xgtkZcv5gEOw8CToGJAjDEJXVDGTsnRkx8bq1X0RYCuys8vlk7Z4cdA70L3STaWLhQZxp5ISNmdbwyes3bpgzlhPFa8rp89mChxA42Jk+bZgonZR5T7IEsTbm81THcLnrpXetgSlQr/d+NRYpuuEgL5fDdavT2isSlKXUrOIguMg3gJzWuSuwxSk8td3d7QLT8VNlq510T1FkagKqhi5rywK7m2CKrYdG2tXsjU3w4kld78HW1tRkUvlE2Z37kzO/niA7jV1aFfO/qAkbEdbxNLXgg/IbSXobsScpE/Zq+6b4YX0Vf9ezHgv1xI3ucVCGoTR0ne8CCfQcrnI6kSVBxHqNSPeW6hP0TOlI/v+DulW0LUaxEdY8ZeckU3q27NDLlwDAr65tuCbEblc8ZR502ECfqg4BcTC4lQKQ+9Sa6wNQlfC+eu2/VBxnmv7f/CoYl4jFGoAs+dxJkssFjQTdJ1aFowFLum6FeoHJcQYxWaVoS0JMczR1w51q623n7+w6NAljibsGspxlqxt5S6igSHYzy9yyLT1t4BxCxVglmB1w0G9zflSK6rO0A11h1Jpqs7wgkIrb5/pPpeqxmEAuwbj9HYCv0fu962+FVKhmZJr+1n9V69rOrNrtJ/0VX6NlYntpmsAx/ao+DslB9WhU90pyfNGbUx1pWRJfUAx1Vv8RiDMqTJNdpHaLur/5sJbXny0mgBAElJAYc6RkOI7RUsKlsyu7AcwG6Z8ckillL0wjb0CJwl63AvmImx1+GewszUzS68sO1mPLmDBGVSbCCTFdwtp/3vHSwBKShZQHBPuG7eCgS8AAYuknCMrHQyj+gzdbGVKf7BBu7IqDcbnrpyv0taIcSWjLtkm9+LXEx4jwittaob0/xgcE/yEaXuSviba+zes4gufjqtAk2s/7oaFLXrXlimdUvZkn+FlsbwALBDWWhIG/lJ7GkF7Eg7sLbulrxFG5XKjGcEc5UzfPkelgpkozxE15ElYUcYKH1N7ec+H3tXZKFxQQ5VGJdbQxUtDIwfXi4DIorBSTHaC9sPSGmrITnXPvQcPpfG1zjDBw+TEN5FFWQ3vYIJjw2jNRC7XPp+WSEFoaZ43mRSjxBhsc15xvkFfKsyd8zOXBWbCSw3RWojLkaer7fWMpS7t2LpVCd8ycUtzXwtUJ6JjDd4pb6DYT75pUDtj+a6D44OuEElFXXuyk3NL9BGo0YORVg+C12+l97yim2G7niboTFXB+oOdUrtY/ZqAreP/3Zr2j5E17Tnj6e94s+VfYLXmGiuaV4SiOnJEw+42TRXDPAu8pskekRtYslab++9j6wG0L8yoX4CSW31Uy4EYHmO/un3ollgvmxtq1cJAlWFFli7zt66xacoMz2tIvRZhdiPNMmdaEfur5t/DSlNk5blADHLuKkE4xcr+CRrhbVHzBYTe26nqws790Qcn/Kphn6dH/WIRWcyYaPpmtx8sXzaq7vF6rZiq9NSevrY2AgiMe/ymCZAGrsS5W931ZBz3lDoLLrlrvCGf8zJfXaD3TtI89Y0bkJu254t+LW7Pwnq1c0A/hC+/5X6+ugCS+pK3RkwMvQfdiJxLA3RbOHNMZGXBmumwkbrSm5S97LtRXV+g7dSFnX5s4YzvCbnGkv68WRhdXezVZGP55/ZoshaxlyLfarRn6NzVZ/p+p9x9sFubBQRV9xs/fOPdcbPKNJWb0jSPUSU41Y4y0j0oa4lWWDE844MqQNeUgQlUcjwiCDQVOml/lM6BtlVVt/KZlVRWw6jrC5k955sXV9d9HRr5lrHOozBWl33kQMGDayG3kRaHJLoSBt2whcAgLEZYtJQqZfPaJwP5ZZn0utbdJHR1hP+0iLTuMnBZLgOM8/63j4gJwqucWnHmB9nan5+hp5d3uCg5fY2unUPEgQXpfRb2i0BkbvLYJjintk9LGDOmb63KfQRe9yjFa7kx3/un4QPTtztCrkaxxYKqdCPswiT73I4FeBxAO10qqpeS55Z7nK0+Mmm0E3qfwLMwjL17qfz0g9MxnjXNOK4uwmUkB0fniSzKbOK8KzgVn3sFY1ydf09Xs+8sOlJAfeocxs3IvCJjVppXSx8oa6yNeSMtpYLOA1au1/iNTInDKl9j9TAZesOu+la6Yv8Q2U2MtEZ+aoUoRu8wqfsph5VbK4ImtWOk+K5WUNVuKeRszehDrRXFOnpusDbYVLEU58YfhRl/MLPDLj6Td4jlL8bfL/uyVlNgaDH6NGh87O6CxSJ8det3LPH0vQGTXwzn7h3znDEhq1gxzlYdiV5Ev1NWksZ0Ogw8sj9FBpy6M2OHJd5wbuUe0hUhVOt5xdGlXR8RmVNtWaJu9hu2LJjI6V1kAnCmzXGa54myBRYGU0zVSMyogvhmgRXjkMET8OC5+LtYIAxE/M7+NrgzkYAP5cw1F3ogjdivjp42+ZwlVbr0RbdOwgxI5lWEbUJ83eHp2UiRoXNzDd/j1AklTvlqkry8r8p9236ImdAopwYzHnAyzGRlWr8b2Zrkk+dm1h5b3OSxAR7jD6mhRcmTZfO8QTmdYx8C8p0v6xi+z9a0WvGKKo43UMhlpH9c0dPAjbQfgNXtf03ndRW489Vrw0wFjRlRcGNb22DYsOnU6xo1itXy7xAcG9MEsorIorD3KQ0bnTvoiLWSfUslVyx3/rO6i1xB9WgiVC7J8YHG+3vLfmF8qzWSdl5eWDW4KyHp6WFkfb16Wln/h5wd6Xc6env/IWc+ABO+XSVL1zj3AhKK3cnfXF+hq4FC1UYjWddaX12yG4OIhV1NNewiqiF9H3+Yz60OK/dORGQzmaeu+BpU3PWVDo8LsriMqEfL+N0SXMhggsrzlgvYlw67BNomHsIWLG9COSNOvCK21TgoA4/w8sdT8pp9l1XKZ6qe7n39yXXPqQNRkKxxR0nV9iK41K8ZDZW31l2YdiVuTOAICXrF865DpKmuxCvMOB4GMlDjCkdQXzmnSo1MWnB36Bhff7y4mzdWCt8AygVgB1vy6QaaLc5GJCIrslmV55vo/hlWZFHrgFpwK02Pa3S+00sVH6JiMmKXg16JXaarKQoSmG5nr7qeq7jKmWkq67Z90TxGocF224oNJ0q24YXdm3RZYrEpuJrMKj//fIme+lqJzxW3uvKMcSjggDywy7tSavvNZ+i7oaNB9KMwt0KuRccQ0pRU0Mxi1YU+MmmT4AlccP200PO6yv29L016SxeYbNCnUXONs5nCD1GU7xfukJgJVGAm5goXdGc6RokVTO1N3yeho1xew7LovcxdcvS2LWAr6yyAFNqjfUGqgCVEKgup2zfuPV2jXysBpuQ7mVOOnjKxOvv2OWKSPEcz+3/U/h8WmG8002ffhuOLhpTZnOPB5PzYOlRXwz+/RrAo+LpATm7q4VdyvrNRg5FJMXV/nXk86zYImirLyEGEVkVcudvD7PO737Gi6KNLAP7228/vfn/z4fLbb13O7QorzEZ5ci3VbcyS5b0X7Pd6wXaEbdQJhkVsJcLX7MTtUtI8B5jY52KTwISZS0WFZiSmAGm5khJgXMT3ggTiA7GAZmvMhsOJT/YOQO/z2EDt9Yldoq6rWaJLYWa5Nip25TvUaydziLXf0mjvaF3zkc5Jemyxy3Yw2ECl8cUm27oXX+9iQczZqKOp3moyR+yxWw12Iwpss1/eExbKR/cTvL/jwiLv9f8Pw1W3KrOb/PcgLJa3fPQekZ1IPghz1HHcXfhJOUHSVudkW3bpU9NktNdZdtAn8xm43Qacuz8yXbesZlPEw6Doa44Zt7Sum7lce5lxddGubYNOXNYcNHQRaGEwnlVY51xnVkU8Yj/HJF5DurWvPjqXRVGJvidqgJ04rnHTqdi9p3fm7zSsUze46eM061Nxu8Ei/3cZjpptcTPYsGMkw8nYDRfuIKcrXTLCZLQs0akseMB+jZUYBh0eO+paFGUmUwnjm/fvrtFvzo+6TUoNI/Jl0lSCm/98i75UVI30bq24yBTtd+pMm9zQcohu0Ie66CyY1tVo6STiQ9oGKmOPEbBAy6McR/ugmkBw7GS4efwBDZhjVSQ4LQs2gXsBlxELkBugVR5tKm0HZtxuVx3QOTZ9rfBUuDMqyLLAKlZZSQN3U+LB+OKTo0+YDNKposDMltF5gdB53AKqBvB8Aa2WEoCVsz8SQC1x9EkYruNUdPaCoHvGYj84vnNbQa3qGR1pkWECg1Hil59Y2FpENN5bgGeLcvWTuDPL6O87ERkxKst11L7rLegW8nGRpwMArziOLjFERsWCiYhFkUPQKXKjRTbP9JoZEl1+iGzO5VrjIn7uShu2MKt00BNEXYjImEgpTpgoqSpmm2gJ7wPYJblNA3yFeQpeYWVWKmlkFj8kBdBXP2XgcYwPmye7m1wusjwFsS3g+PlvRGQFvsuMieU26AK2HM1pgkehYCIR0kykQ7rkOuMznsUOi3Zgf58QePTO4C3YsXshtmHHruptw/45IexXCWH/a0LY/zMh7L+kgW1kyfGMphApDfT45pnIioqD8j3bJHgna+DlbQK9pKg4WxRlGu3bapmYL2InIXnILIVSoukXEt83IjLtEhITnKBWJI01aQGnsSb1RldlglmkRDRl1UlMVSONNT3oXQIRYqSxhlkq2GDWJAFeCXYnsJCakgRMuHplqZLoUVi9kqVZUpwncKvJoswIT+DDtoATBEkArpptTHy3qIWsk0AuqyxBTIMoZhjBPEEBkc7wggqyiZh11YYtMN/8SfNZCrxXGbQBTQLZtYNJg7VLrE0CfbYoV6/S+KB1NmPmL0kajRGdxZ0V1wOsZHRRrZNcc4BKiYpf5aadjz/arK0WYGqWzs8f3znigIPalwS46yYfr4NcC/accZrChtHZPMUhsnnM4uwu4BS6gc5YCUmKWRJRx8rVT7k25aCZfyTYWpEksDmb0xRmjAZHc0FzFq1gtAubiTRcUsi84lQTmYLaHjhbJJBNstRrbKLO/G9BD2WQRwGs6IJpo3B8T8gWdgKNT9EyFalVMlpr6ESuEslXl5nvWDwBdKMoLhIokq4UKBXa6ZTr9VIynbkJs/Ghb7DCSRg8HymEjQF55ebbx4bLtMEi+pzjXJtZpWINC6yhUjcrKAXUKjqu8fXouiY5NliY3DCPP+z62E4Du2AucJ7HvgMsjx1WrVsHJXiLWJERJWWRpCuRBZzATGNFliY50nc8SkHm8jZ6e6ZSx29ZykpdKhYZKMeGmSp69hlngsZrsbOFqqNO1GngQvFtfLcWl67raTbnMvpz3gBPkPJvbd7oUscCTSBxrA2dANXouQlcLpKwrlgkucClVLEFWDGrFimuWcE0SSEWCp2EYVPMgRDUQHOl6HCjy3DXADp2xp+DGjsdT6zXsS2QJBVl0g2Ajm6JyviakVRskQXmcZ0Mdy2oiv9mlZkbyhsdbNTJ1FuwbsRrEiZLULjpZ+LEFgYebGxpUGbOkRQdXay1/TAjy1h1/gPQ9K5k0QMBJVXFQmFhBj13Y0BeJwEc/+l1ncg+fepNAY0AWMlFhnUZcWBAG7TCsaEqinkK/U5RAnRwXUcTAY9PZAs5bgvXFmSp8gQYx3dk6gS+Ye18wwnyATSNnQjgBh4nME40/RKfAUINWqNBTWBKabZIIHh1GdvLphVJcQ8UyaMr0lqRUFfcCIBNvBFbbZiVjt5Vc0VE7EKJ4LTYU4G6Jp2xt28WJj5bOaDxI3rNTM/YcDdl9G6tVT5LkodeKZ7gLaw0VVnOYle9JxlbUUeGUpDBEG1wEdsbvMqY0AbPE2gGK6ZMCjV8VYoErZuMVJWI6WYNtUULdBR9UxmJPlQCDZZuskcSDsv7jDnL0bmiOTPoHKvcdzPU0P49jI6bnJWQSmMTQgEMDNFH0N+ASI5CpTpNPgQT6Sh3WZRcbuhgsOBe+s1lFa2p94E8ZmnofEYw70zRBb1DBe43WtjGYsWi6g8DSY4kZxqGM9Sr+6OHBkpIV2UplUHDxqMIrZfYIGZQqeh8jBVOSMu9zxCKEOG91dGggJjwnd1H+kJzJlJP5G+haldr46mRkQtqllSdbb+vl7IavGgICbqiqhlHZCQqsdIUvaMGw0Rwd1dxQ4Knb+VCv7h2Za/P0IUf8fUcmWVgShE0A/5A/ehjQFug99T8zoygOnzOQ6ZOQrw5jOxubhEs7jarKVZkecYEC+IHM3cn6K/dE58wCwOSIV5wXAmY9buoYI5r3cQ93MC91699x57St+Nu9tQ04fbzi0eMfXsQWcSapsM6r8Ky6CO9M3ArxtwFU0yjHhFI28F172FCteAjEy+he27CceDQP1dTgxT9UlFtdjTtPj5b+f698p3KAGN53KpOYvc9Uk3eadedsgsnhxHExjp/hw7t+nVw5zFn/++fb2gXu7qohQKsHeYNsBriJfHek4Xt4zLDmiKXrt1ggwa3qjkl/4uHwVc0o+AbzKVy7euDZEQIa6QphXFnePe8KoWFxmSC8b6DDtNuaQFq75ZpSKVgAtoupEuqCubUjamQ3i7pBnOwFeN0QRGnK8oR1pothDu47bz+MOtDS+YHlN+w/g5Onz3IpGeLWSXYl4r2xyTi8OVr4Xtcx8TjpqDUGg3L3YUkUggKuRVozcxyTFAgFKgMaTR2RY8qL7q3aWHJCfKkeaK4XDCCObIYjJg+gMXDYgdLjYxpfDjalcuNDqPXSmdby15Wa+wHHnOGdbaUyW0CZ8Q15hrMUtkONbJSsT2CJ9wPALlLY7GFN80PYiGcYnX2hmtpDfHOfbuAYDn61f/iDL0Rm+ZfA+gGbHktDML5GZFFWRmqwmI4iRvfbiydefZN/yxgxmLnQJj5Z/Xy+x/+Ym3fi9Zx1BT7Joi259MsbsTsUMcN3lCF/rXxyekXHg1ALnzrY9f/pOd5scW5w/U7z+PI5OV9su1Jf2CKXecMvf/t46XdO1XUOU/AX5ozTRQtsSAbq1V69Yz3c0EQUOg5+vjuNboS5seXz9HV+4vL/3qNPl0J8+on9HS93CBBmVlShchSaj8qTSpFiYFv/fDqf/9/z54EKULNMqGM69MDZOpZgcPjeHRi7rvnNb9xvHhVIxW+4vnjQrotm/ZgfmTDuIMf+BC+PcV0a518ZspUmKO3b94Hkf1TCprOl3UcZ/wfKehZmLYW3a9GhMJG9gtPOILH+AbvOIcFNnSNH2BEOnD3NXqT5wr8tI7LQ+g0Ty8pymPjnKfGQq7O3127V2k0PFZgPWH0o+NUcpqqf7vR1bVFZcT7ZWl45CSIKDS0a4/TsNbEMjdda1oB0UIX5zmzX8Z8G7BtzfIPv3MTMoA1CeGCS3/DL7osMEBlm2udRK879EnD6L3H8Foq04jkgdDNIcAGB8DMZr/k1RPT3u2HiUX9mNTbejdGeEFDduNUXlyPHVi+WGtJmFU5nd9ooOMgK5cVFgt61phORIo5W1SK5mi2AZhU5JA1FJYz5ZGtBwZFoyPacnDReYJ+Bzyi7t8u4YruAFC0kIZmPrM7fp5RfNLmQmc4c6n4CUCXRqUBPk/AEvME1cI8xXVI1f+kTEBUnGe1Jy6dWt634O0+zvqrtZ0JD6DBXpolVYIa9HFT0ufoU/2MvQUH2I/ounaADV6C38Y0tXpUzwTKxIhpXCPt/eLPEeY8qEyU2y9CghtWkJi3osq+gUwYibSBx5wJ9OlqVKAQSJBNJq+ii2wLVJYJxr5ZwIrq2Bm9FmyCEhf3IsZORQd/ewJs3WiFjFOxiD4pEnC2ykdCLXREA3UqD+atAIxABNIJ5gijX6RaY5UP53Qj9GYByV4KYXvj7yCXbkbNmlIRVj0jd028b4xbGszboTqHDIKW8ZAZMdghEz7PFdISCmasWPIjNsJbXHEspojjH+CgrBNEWi7KwQa7LsttJGVlLdgFGLDdlyd2pJIS6EKwitcP7rCIPVaGkYpjhaBfNKqReHp59/qtXMj5PDz9nZLMLGny4+0g+9Eu6G5jC+9Li7dF901lllQYnyw+irauYnZOOCyhxy05jvonTdUowrIyRE5Lab/kOMI3FSFU6xGcofP4cc3Rjks8AbyQVXEXUm1QoDBhgNsUwqmDI+3haKUSBPh0KYV9V6zcCimHzQ/RQFHq7moVrx/dyLuJketaCjUDnNG82Y/3w/T0YSaQZqYKyE8ExQXUi2gPdYk1wrks7etilpQpJNdie2SOcAbfSSGLkbxamMmhmWtRP60SYZV7JnIrf6TSDQEw+oVxit54xM4GZDjE2Suajbk7OZow3uz/QdIVRklw47MW4lIhtMcAIWLWu59ACJevd+PrNWJTYjwhdCZTVg8ENj+jS7xisgLtksiiVLJgIxmKdGrkLgWecSgim6Pz3bgxsWrETkIk+xh2tE4URKCDYdThMkcgGFi/wS/16bZe2e19G2W7bZllJUy/nC22Rp9DGXhGjjHrD9KC4D1eUEEVI/WWgCCQ6NdPLWBmCU9taLYb8siekR/OtFHjwc96T8e03XqwPb3cvSevXri1Eu4raJo2RrhhBdVWrjttT9GSjgaR/ClEawqx9yCg8eCJx6AOZK1jenc/GGv9eNiefsh0tCGnB2/NO4z37XCwN9jxViAcIAy+3t293Ls7NenZuYsWZW9q/8lF66U6jQDZI8cbAfL1suOP+48s1miDaY7sMPmoJpUgMe/YAfJjUnaMubcBMzZKPZSg9fzU0St3KrPMCmqW8gGiJLjjSUYODf+10QOHXkpKJvU67YjqfJDc+2stIjv4MpEn5L/Ofv7+e/T07cWb62fogmnDxKJieklzKIUP4sLlQibvC7QrEgbZsnOHhz9m+OJIxpiSib2Ku+o/7amGMGhuDHjkow19vs91IZD239T9thx/gFMoZopFqE36NlMM81jd6Xob+YBzVmm3ApIKaVYwjpUTT1Zs2jtE4F0Pl1fBPdcsn7LTSDtT/pNlhNqL2OuLub3k6eos3ohddx3CGr7SsOX/9U4i+GTAC95xQ1tlGXnYlSlVysSAQcgGSC3VAgv2546sapGOFQ4l9hGUbvPUCLnnTAVrSRN1/fnFLgevhWvx5XoXdbKaf6WYmyXBiqJS0VwWTOBgwV1LPF1jw6gwem96PMdT7vYtftDNutaPtEzEuPbqPLGCq8TKQDOk7VZ3i9UJmx15YXOIRJ3TnCpsaJ5FSyrbwR9W+PxSr9gEz66VXLG8aR7mv4fLkntNdcAYvvmPfda6Om1YwdlukuUT7bJZ0vf6M5uRbQaHh0Lm5Iq56Pmyr7iPtIBrlM6YQ8Hvq3nSO9CZWj9qVUIvAht1OiporFgjbaRyEt9CK6jBsNoT+NaZ/daT8O4LluecTifl3sF6h8q5wPG25N5Rcq4ejzHNdq/9aq0OQ2JTR2efo5Jje2T2fZYKUUHUphzz8kMq5AT25AEZdKqxLX+V2qB3mCyZGDHpcpxIcnzTp/UnAZn+paJWfFj9yDU502fobY5L9Bn+4fSjXApXd/rP4eOJlnhFrebEKVboS0XVBkEPQl1KoWmtUYWLU+1+M/jNNPLS98AjFrJidRdI4bbv+vKN41lvaQJUtwz0wTdHPRRTmPKU1mHW5/G6tXSniZG1Df3DyzRSlRBBO1Y/b14eF3l2baRGauw8xMxbmOkPAqM1E7lca6RLSticEfvJ81CdoM+THV4Quz2H7zbnBj2FjrBUkO0zBKHLZy1qoUrAO/6WLjDZoE+62/i2icAW/ULa6Nm1doUJDPaR175tagEqUKsGTGZfxAHFmz4Ager/TqUplPMMydfddnqFeqw7r1OvAzuGHQYZzf/miM1Ok9c7tlWf4etd77Wsu4Stj3cBHe5mGoddEzDons02IdMdw+CEwg0p9hc/Q9lAzJGAoxVusOWczpnwvnoQTtDVr8DlSNNBwO6oQrFEuG0dMD31L7ZgbHy2qffueymN9KZsfNjGYLIsJm6Bv10VCI4G1lH7OJIMeZkxEW+CWNS7YbcMRYVpH8+AkGqX7cCxuDba2/L+wNTOAdZp3749WJdY1Txl//x8u5X1kg1aqSN7O6wt65LfD9qeiT6zxLW1kGqT7sD/qkss/ra3Y0yNSLeLeq2eh54mS5a/vgDoe/b2YCrRYFd1v/XduxrlgowKo2R5jOjIZTUbOBcO4nG/prW26Z5yBMDRVXdMew/PZVFisWnuI1w7GKfv7JUVVfYZypiYy7BSgPVt6hqhPfKjZ0XWmK1p2q7o8y+pcgR+qTjfoP+sMGdzRnN0AXXPzjkYRGVNZxmR8pY9UND9dzpDbv2t/Yz5mDYfvdvsNhxeVgZU7iNHmO6/6x+aJfyUHe+Odj75M/RxU7qtbz0HljjuBMcPT9F5FrWZbA9ti4NzRKgnOtS2to/MFK66RrnsYuc8i6VUtbcfQswf3o4ceatXTmR2qmlRpp1DtIMUduW9nvsaTSVlIk2ki5Rdx54HKrEJuyaJyLCOGe1vAVa+nD4y5ErxiMfcghrxVBpjNKtULG9IC6amKsOLeDblFnT056kLOmr6Yxe05/oEgoXeGSpAtYpvnFj40bi5UfSWivZSZWJrVG6JKWoJOzL3IywL6tUL/9/nHoUX/j98XlPI7Y85VeHsPL+dB4yeu820g+fgcW2NWhtsJ/cD0axJxcScKjUSdx3ue5J9tRX/vaQPumcnQLLuSzxvHUPgSkFYWya9UoElJmO/Sxe3t2z3ETKIVftP/6DDBK3xgZ+sXFI1jT/C6uw+4+npOYx+fIbOYf0walSZiZqljND5nCo//JN2sjB3NOelSUPHLUK2Dtwu+kS3OkXvPGn257Feyfu3RgmfNrphf4a9New2kUy5+sclEnQhDXMHWC6xHpkApcnUbYVaR+kWHx8uaI862QSoQYJLj8fqxul1/U04IUWzxRQVFd3+Rs3Uw4+jg5atNGFaV9GVToAMyVLpvHWnxVAAQ6pUUh/o4FDa0vPSLo5uIDi9SzpNkiHRdAb3UeSnN5DaufsxaknP45C8v/TcgeO4CNWaZ6uUL3o/pOod2UFk8syyHq6it2nUqQCzW+ot6kTNDb7ZjitpP0ggW39CGuJ1UqGrmzf/eHeNru07hX4TI9NXttgmqqQ+BtuPaxnGFsQQWVJyq49yIh8mhNP2IAsNnWv6dTYtwiAN1I8g3ErBHVouVWzQFPIBlFyHR9MVZNRoAJwNNtVkEz7bWK4wZ7ljxAASfUE4WVfrXYIQKHZLN7ovtiNxfp1AGhn20phSZwxm0CYBDUeZgiAEP4LbxBairnyRipnNnhtFZFEk7RN3IN4OD+8QCpfgr5mivG9pxnaxrDkWmdYPNfDWruxk+O9+t3WNVhBbV2qclZJNkVYdQthhgAADQCpsDQBZyRILMWickbrdlF8VEBmJ2U7Utrl5WPzMw9/fvnnv370XveWbB8VI1ff9R+/ZxvRttpK8SkWAN/UcZ+Hn3DSTsetxvpVgRqOnDgn9DLp1QGFvPVG3Bx4B0sHd8CqRNHvrcf0kmPHpAmfdooMVVZApMK84IlIQWhprKN+4Mxxpr7Bep5S+jvDWYK9HaFtES6kMkpa+v/77m1AKbpDssflOqsX0CZb9AoOOi3WGXbOTYKOYv1/+dn11jd7hu4KJvBnrHT5Wu7fJ0zA7QxRHtuW3Mdjdrm016lO4ZDF6erarcszm0xVsPnQRfr3l5GpHx1nmpfLVhe/S67HYiSGf7lAeuFdAvePiv33dcFOYI/KhJhn7doO/xJrQD5Td6MdVgxXfBHULV9z7HOkqkKKONfqrNkqKxd9mHJNbzrSh+V9f+L89bz5lYk5J+KM5U3SNeVCRwTPe+g3CIkdaohG2VHTBtFEba9lPKSxKbJa+WX+DA+rjMEASnFJToekKoV29FpGq1YW80ScbzKkwrZyUGm8/kPGsmaZ21rv847iP4Z3TOa64yeBOvEZzzDulyJ0tdTP437eSI+pJkduR8duyNaPwfM4IDBKYUSqQnEHfiFZDr+ZcNL7HZvoXe89Whre+cRlbrEVidbLQqdskjUgUhdeooFrjhe9LRKSV3zDALKRIvpULdEGJzEfCPh5WdB+V6/kcMYGph/CU0giKMO2LJueICW2wMDUaYRvfsKMe8Xz4TgVVcbiHzFq3xtU5bccToKW1bWHC7u/MCKp1ffr7pyAIuqKq3aCixEpT9I4aDJq6r7ltlnr6Vi70i2uXVPtsAP7Cp4Nt1QqMPlAnLByHixaaI51k6CqJC+e0aHOhF2mVZ3/G7/w9v7r4wQdcXNu3rXUNPQHuMDGIy4U7r2FfG9gdTLL23ALf0925Q/b3/mDPRjljAPooTglxxgDyOKeMHslq2jN5+f/OZPeZ2FXTHMhp11fO/siCva4eDXarVKHS01BTNGVW7OlkS3X/T8MMbL90BfenIYernJkM+lE/RvS6htMjQmwZcaJuVMSYOA6xtBpTLTkeL6fl9KhhsWnJNqc0T10EMh62aLdNdI0kaT7QQwZKwmlWRE8PGUDfY0WMU3H6OvP+YNwg+Ry5BtuMRD4UoOC9yUemUKt9dKBRo1Wzf/+nTdeoPZeC2McBG/nYLdsRcQNN6hKKwzZ1z+0yLvmldZ/fyoUf6+qrGKCXnDVBFPWCarD1ObujOdIUJu12ftxdQ48bLPUhDGCfbLA0hzAAfa9DGXoC4/uXjmPMwb7uQZP70SBii4UdfPlrnVfqOZL3OVJT0XQe5nKhQ2zT8iF9PfRlxzDY4EejhL26Xv207Qc4ct37xB3s3sivlbirV6nJ++r/XvImrn3yNO7LBedIa3vLcoTRgq2oaJxkX68iYEl0nP8irQWSP0bl7+uIaIw6NGS5yRT9kuCs28FDOGDYt2/md+l7il3DRXruvdkGuwprgocSZEbr5NFPV8L88ApJhX7hEpsfX3bTvIgUc7ao1Hh+y3bfx6i7X/G+IQz6WMsmwTKeoGfGWHZMXU30tTsYpFpjlSdT6nZPqncKyeeOvoeRohwPU9Nca1X/iHq0fTNM4FS97fIhFVswgXn9m662socOqfSvHYkRV9efXwVIgILdZFEEEjQYDakc4/XZMupQcTz29VlSnCcsr++YdrAUuro4JUrq8G0HSwHMcbHSR+1k4yRL7mfDTQ7uVtGCi2JNl3PJOfRN/RoFsKXeA+TcWJ5jGhFHuno8XEtRfSuH4yzGCf0ILb6CzB6LqlpIberCvdlmcGjNJC4LULOi5Bt/TvbLkMxMMVkizXKKnn6PzFJV6OXPPz9Da+xHCdWr7KDEo1BeD6CEn6uTjBTkq+EKN1Sl9ik0fVftVdZBCOgpnskVbRGDhUt0avGmjaK4GL0/5KthmwcmFc3ZUU0T9hHqm5Dm2DgW2BwxU/f9AZH+wrUJrZEejrP6J4J6kQ1V6CW6FASXuuK4aVZ2L7kegn5i8COQWxla5ceX6N/sdp+jH39E/4aIVFZfdj0H6mFq/4Ob/2W/yDTqEiXc/kLInD5aW1esaUYw5zNMbtOXPuVUSFOPRgO7whKxrnkB02RsKh0wR/JmRsAy0HAbc8DYzbE3UlnNWmyc1mE/aDWjCCGF0FxWIrcvDIeBDBo6AhyWvNi9EQPIMWKB/jrsCBuNnMKGS5w/lnfOo4M0+xOGUSpGAlaHN4XbXwZb2D33tRC2zz42W41WzutjO0O/yrU9mqHNyQSSyhpjRqJbSss9RHsUL95XQjQ3mCJbpRx4fllLHhhL5eZTC5jE37ILV0zByNSri67vXQRcHO2Z7kAMtwt/1a8ukLLSWoNDZThbZHT6f0OJZPXMD06J7jySkXy5JKGgoeDfNr/6AN3wmxnNRFHsBwGNCEr7vzoQ8xUEXvxKmS45S9295NGa85qlKoQ9MUX6uKZRh/I73Dr7BtQTgTzX1VaLf0L+e0QYnXgZjAuaJEYPI4CkQtfnb6697kuwsORhRSlVX+NF8ER+dWkQ1eNwf3xyTxUY4qFRt2hoylfbn2wNdqfngGV+hl7+/Aqtge4FxQJhzsO+grr6eY62/iO0poo6sNggTrE2SIpeuUiXiA+uJn7dRAzc1RRhW0+736XKgXCQ1UTJUkguF5t+IG7O1ECLRehnRJZYYWIcESm0L7JYuAnuqBI+p4d3fOajFbWxC7pdoD5lEGHXtAVrURRWyZSiDiMovB6VaSBZe2olJqCxuhiF8D4HSUilaojaYJFjlSMhVYE5+zOU3ytVEaRP7rMcjibRYbPwdhBpi3WDzAvO5hR2HDDwNSVS5CMK9va4M20maGgf2hATRBYlpybIAKNOVAwK/HijaW2wMg/EyDd27SA7j7FylzNH2a+QInon5HyQIHFy0wORPxDhL0WeguwW5J9SPFD3nHr1WsV06bUf+xQeiKhkN/oNgmHcfgS5b4dbY5fvygMLnO+pzLbpjwI/HaSiRKqc5uneQZ9k458p3axY6xh1pk3zxXZ8ffhaKVmcAdQKivI1oQIrJp1aX1TcsO8MowrhsuR19cu2l02BBV6ESnMR4hDeqe1Fh5TDVSNmnmgk18JFxgwuyr5n0GNcT00a3j6jEVkya93InOoz9K7SBsykNlDXPWskLxcbeuQh7RRg87nFe0Wn0ITgkOsFHe3c0DRBHENgq1rnbMVyq9kAP4QF2U0tyD72iBfe5F3J1GQ73J6niwXdWU5khm/cZrUVelZfs0gBg+72jUY89D3dvmt5djZYcttdrYotgYroozgb+se+KqBBfqloNRkrWe52XLSVj2sMY0+rdgOuNpolIBdr1END1IhKQYegCWTaojAJXt9FkQLXMkuAapml0J7LmKKoCzTWqI8t1AS6UusVeRgTsmc+Bt+YwXN5rzfnWLG5T64dEyzYPhC9bgixHUGYDJT4GIq1rvgDNc2XlSGyoC8cDo3x4ge4DDgEC0+CjgE5wiB0RRUzqVuDjnWf9qv7IsCx0aQ9l8/Eg9vcK91UulhoEHdyo+63hk9Yu3XBnLGeKl5XTp/NFDiAxsXI8sFk2GYSbBDv0BSZhIfwuWulty1BqdBvNz41luk6IaDvV4P16xMaq5LUpdQsouA4iLfAnBb5trtwc3dHu/BU3GTpWhfdUxSJqqCKkfvKouDeJpr8fEAlW3MznFhy93uwtRUVOcxJ3iu35OyPB+heU4d25XA6bRux9LXgA3LDPOCdiDlJn7JX3Tejk2C9mPFeriVucouFNAg3k9TCCbRcLrI6UeVBhHrNiPcW6lP0TOnIvr9DuhV0rR62/W4Uf8kZ2UwxbWdELlwDAr65tuCbEblc8ZR502ECfqh88/+wOJXC0LvUGmuD0NV2VEBdXZXn2v4fPKqY1wiFGsDseZzJEosFzQRdp5YFY4FLum6F+kEJMUaxWWVoS0IMc/S1Q91q6+3nb2QocYmjCbuGcnwwoWOSmwOGYD+/yCHT1t8Cxi1UgFmC1Q0H9TbnS62oOkM31B1Kpak6wwsKrbx9pvtcqhqHAewajNPbCfweud+3+lZIhWZKru1n9V9JPcfRml2j/aSv8musTGw3XQM4tkfF3yk5qA6d6k5Jnm9nkCa6UrKkPqCY6i1+IxDmVJkmu0htF/V/c+EtLz5aTQAgCSmgMOdISPGdoiUFS2ZX9sMUc1G6ffRD01CcHveCuQhbHf4Z7MwP1djKenQBC86g2kQgKb5bSPvfO14CUFKygOKYcN+4FQx8AQhYJOUcwYR5RvUZutnKlP5gg3ZlVRqMz105X6WtEeNKRl2yTe7FbzPNhPBKm5oh/T8GxwQ/YdqepK+J9v4Nq/jCp+Mq0OTaj7thYYvetWVKp5Q92Wd4WSwvAAuEtZaEgb/UnkbQnoQDe8tu6evWIEMYXPgclQpmojxH1JAnYUUZKxxrYPWeIBYsRQ1VGpVYQxcvDY0c/DRpWRRWislO0H5YWkMN2anuuffgoTS+1hkmeJic+CayKKvhHUxwbBitmcjl2ufT+mmTz5tMilFiDLY5rzjfoC8V5s75mcsCMz+IF/ZdL8TlyNPV9nomGmA/GA3HxC3NfS1QnYiONXinvIFiP/mmQe2M5bsOjg+6QiQVde3JTs4t0UegRu+3m4fC67fSe17RzbBdTxN0pqpg/cFOqV2sfs3WmLzdmvaPkTXtOePp73iz5V9gteYaK5pXhKI6ckTD7jY3Uz8LvKbJHpGbzhj//vvYegDtCzPqF6DkVh/VciCGx9ivbh+6JdbL5oZatTBQZViRpcv8rWtsmjLD8xpSr0WY3UizzJlWxP6q+few0hRZeS4Qg5y7ShBOsbJ/gkZ4W9R8AWE9+bUu7NwffXDCrxr2eXrULxaRxawZ3zvvPFi+bFTd4/VaMVXpqT19bW0EEBj3+E0TIA1ciXO3uuvJOO4pdRbcdINrnZf56sKP4EZPfeOGejalK/q1uD0L69XOAf1QA/69+/nqoj3ftRETQ+9BNyLn0gDdFs4cE1lZsGY6bKSu9CZlL/tuVNcXaDt1YacfWzjje+Jxx+fNwujqYq8mG8s/t0eTtYi9FPlWoz1D564+0/c75e6D3dosIKi63/jhG++Om1WmqdyUpnmMKsGpdpSR7kFZS7TCiuEZH1QBuqYMTKCS4xFBoKnQSfujdA60raq6lc+spLIaRl1fyOw537y4uu7r0Mi3jHUehbG67CMHCh5cC7mNtDgk0ZUw6IYtBAZhMcKipVQpm9c+Gcgvy6TXte4moasj/KdFpHWXgctyGWCc9799REwQXuXUijM/yNb+/Aw9vbzDRcnpa3TtHCIOLEjvs7BfBCJzk8c2wTm1fVrCmDF9a1XuI/C6Ryley4353j8NH5i+3RFyNYotFlSlG2EXJtnndizA4wDa6VJRvZQ8t9zjbPWRSaOd0PsEnoVh7N1L5acfnI7xrGnGcXURLiM5ODpPZFFmE+ddwan43CsY4+r8e7qafWfRkQLqU+cwbkbmFRmz0rxa+kBZY23MG2kpFXQesHK9xm9kShxW+Rqrh8nQG3bVt9IV+4fIbmKkNfJTK0QxeodJ3U85rNxaETSpHSPFd7WCqnZLIWdrRh9qrSjW0XODtcGmiqU4N/4ozPiDmR128Zm8Qyx/Mf5+2Ze1mgJDi9GnQeNjdxcsFuGrW79jiafvDZj8Yjh375jnjAlZxYpxtupI9CL6nbKSNKbTYeCR/Sky4NSdGTss8YZzK/eQrgihWs8rji7t+ojInGrLEnWz37BlwURO7yITgDNtjtM8T5QtsDCYYqpGYkYVxDcLrBiHDJ6AB8/F38UCYSDid/a3wZ2JBHwoZ6650ANpxH519LTJ5yyp0qUvunUSZkAyryJsE+LrDk/PRooMnZtr+B6nTihxyleT5OV9Ve7b9kPMhEY5NZjxgJNhJivT+t3I1iSfPDez9tjiJo8N8Bh/SA0tSp4sm+cNyukc+xCQ73xZx/B9tqbVildUcbyBQi4j/eOKngZupP0ArG7/azqvq8Cdr14bZipozIiCG9vaBsOGTade16hRrJZ/h+DYmCaQVUQWhb1Padjo3EFHrJXsWyq5Yrnzn9Vd5AqqRxOhckmODzTe31v2C+NbrZG08/LCqsFdCUlPDyPr69XTyvo/5OxIv9PR2/sPOfMBmPDtKlm6xrkXkFDsTv7m+gpdDRSqNhrJutb66pLdGEQs7GqqYRdRDen7+MN8bnVYuXciIpvJPHXF16Dirq90eFyQxWVEPVrG75bgQgYTVJ63XMC+dNgl0DbxELZgeRPKGXHiFbGtxkEZeISXP56S1+y7rFI+U/V07+tPrntOHYiCZI07Sqq2F8Glfs1oqLy17sK0K3FjAkdI0Cuedx0iTXUlXmHG8TCQgRpXOIL6yjlVamTSgrtDx/j648XdvLFS+AZQLgA72JJPN9BscTYiEVmRzao830T3z7Aii1oH1IJbaXpco/OdXqr4EBWTEbsc9ErsMl1NUZDAdDt71fVcxVXOTFNZt+2L5jEKDbbbVmw4UbINL+zepMsSi03B1WRW+fnnS/TU10p8rrjVlWeMQwEH5IFd3pVS228+Q98NHQ2iH4W5FXItOoaQpqSCZharLvSRSZsET+CC66eFntdV7u99adJbusBkgz6NmmuczRR+iKJ8v3CHxEygAjMxV7igO9MxSqxgam/6Pgkd5fIalkXvZe6So7dtAVtZZwGk0B7tC1IFLCFSWUjdvnHv6Rr9WgkwJd/JnHL0lInV2bfPEZPkOZrZ/6P2/7DAfKOZPvs2HF80pMzmHA8m58fWoboa/vk1gkXB1wVyclMPv5LznY0ajEyKqfvrzONZt0HQVFlGDiK0KuLK3R5mn9/9jhVFH10C8Lfffn73+5sPl99+63JuV1hhNsqTa6luY5Ys771gv9cLtiNso04wLGIrEb5mJ26XkuY5wMQ+F5sEJsxcKio0IzEFSMuVlADjIr4XJBAfiAU0W2M2HE58sncAep/HBmqvT+wSdV3NEl0KM8u1UbEr36FeO5lDrP2WRntH65qPdE7SY4tdtoPBBiqNLzbZ1r34ehcLYs5GHU31VpM5Yo/darAbUWCb/fKesFA+up/g/R0XFnmv/38YrrpVmd3kvwdhsbzlo/eI7ETyQZijjuPuwk/KCZK2Oifbskufmiajvc6ygz6Zz8DtNuDc/ZHpumU1myIeBkVfc8y4pXXdzOXay4yri3ZtG3TisuagoYtAC4PxrMI65zqzKuIR+zkm8RrSrX310bksikr0PVED7MRxjZtOxe49vTN/p2GdusFNH6dZn4rbDRb5v8tw1GyLm8GGHSMZTsZuuHAHOV3pkhEmo2WJTmXBA/ZrrMQw6PDYUdeiKDOZShjfvH93jX5zftRtUmoYkS+TphLc/Odb9KWiaqR3a8VFpmi/U2fa5IaWQ3SDPtRFZ8G0rkZLJxEf0jZQGXuMgAVaHuU42gfVBIJjJ8PN4w9owByrIsFpWbAJ3Au4jFiA3ACt8mhTaTsw43a76oDOselrhafCnVFBlgVWscpKGribEg/GF58cfcJkkE4VBWa2jM4LhM7jFlA1gOcLaLWUAKyc/ZEAaomjT8JwHaeisxcE3TMW+8HxndsKalXP6EiLDBMYjBK//MTC1iKi8d4CPFuUq5/EnVlGf9+JyIhRWa6j9l1vQbeQj4s8HQB4xXF0iSEyKhZMRCyKHIJOkRstsnmm18yQ6PJDZHMu1xoX8XNX2rCFWaWDniDqQkTGREpxwkRJVTHbREt4H8AuyW0a4CvMU/AKK7NSSSOz+CEpgL76KQOPY3zYPNnd5HKR5SmIbQHHz38jIivwXWZMLLdBF7DlaE4TPAoFE4mQZiId0iXXGZ/xLHZYtAP7+4TAo3cGb8GO3QuxDTt2VW8b9s8JYb9KCPtfE8L+nwlh/yUNbCNLjmc0hUhpoMc3z0RWVByU79kmwTtZAy9vE+glRcXZoijTaN9Wy8R8ETsJyUNmKZQSTb+Q+L4RkWmXkJjgBLUiaaxJCziNNak3uioTzCIloimrTmKqGmms6UHvEogQI401zFLBBrMmCfBKsDuBhdSUJGDC1StLlUSPwuqVLM2S4jyBW00WZUZ4Ah+2BZwgSAJw1Wxj4rtFLWSdBHJZZQliGkQxwwjmCQqIdIYXVJBNxKyrNmyB+eZPms9S4L3KoA1oEsiuHUwarF1ibRLos0W5epXGB62zGTN/SdJojOgs7qy4HmAlo4tqneSaA1RKVPwqN+18/NFmbbUAU7N0fv74zhEHHNS+JMBdN/l4HeRasOeM0xQ2jM7mKQ6RzWMWZ3cBp9ANdMZKSFLMkog6Vq5+yrUpB838I8HWiiSBzdmcpjBjNDiaC5qzaAWjXdhMpOGSQuYVp5rIFNT2wNkigWySpV5jE3Xmfwt6KIM8CmBFF0wbheN7QrawE2h8ipapSK2S0VpDJ3KVSL66zHzH4gmgG0VxkUCRdKVAqdBOp1yvl5LpzE2YjQ99gxVOwuD5SCFsDMgrN98+NlymDRbR5xzn2swqFWtYYA2VullBKaBW0XGNr0fXNcmxwcLkhnn8YdfHdhrYBXOB8zz2HWB57LBq3ToowVvEiowoKYskXYks4ARmGiuyNMmRvuNRCjKXt9HbM5U6fstSVupSschAOTbMVNGzzzgTNF6LnS1UHXWiTgMXim/ju7W4dF1PszmX0Z/zBniClH9r80aXOhZoAoljbegEqEbPTeBykYR1xSLJBS6lii3Ailm1SHHNCqZJCrFQ6CQMm2IOhKAGmitFhxtdhrsG0LEz/hzU2Ol4Yr2ObYEkqSiTbgB0dEtUxteMpGKLLDCP62S4a0FV/DerzNxQ3uhgo06m3oJ1I16TMFmCwk0/Eye2MPBgY0uDMnOOpOjoYq3thxlZxqrzH4CmdyWLHggoqSoWCgsz6LkbA/I6CeD4T6/rRPbpU28KaATASi4yrMuIAwPaoBWODVVRzFPod4oSoIPrOpoIeHwiW8hxW7i2IEuVJ8A4viNTJ/ANa+cbTpAPoGnsRAA38DiBcaLpl/gMEGrQGg1qAlNKs0UCwavL2F42rUiKe6BIHl2R1oqEuuJGAGzijdhqw6x09K6aKyJiF0oEp8WeCtQ16Yy9fbMw8dnKAY0f0WtmesaGuymjd2ut8lmSPPRK8QRvYaWpynIWu+o9ydiKOjKUggyGaIOL2N7gVcaENnieQDNYMWVSqOGrUiRo3WSkqkRMN2uoLVqgo+ibykj0oRJosHSTPZJwWN5nzFmOzhXNmUHnWOW+m6GG9u9hdNzkrIRUGpsQCmBgiD6C/gZEchQq1WnyIZhIR7nLouRyQweDBffSby6raE29D+QxS0PnM4J5Z4ou6B0qcL/RwjYWKxZVfxhIciQ50zCcoV7dHz00UEK6KkupDBo2HkVovcQGMYNKRedjrHBCWu59hlCECO+tjgYFxITv7D7SF5ozkXoifwtVu1obT42MXFCzpOps+329lNXgRUNI0BVVzTgiI1GJlaboHTUYJoK7u4obEjx9Kxf6xbUre32GLvyIr+fILANTiqAZ8AfqRx8D2gK9p+Z3ZgTV4XMeMnUS4s1hZHdzi2Bxt1lNsSLLMyZYED+YuTtBf+2e+IRZGJAM8YLjSsCs30UFc1zrJu7hBu69fu079pS+HXezp6YJt59fPGLs24PIItY0HdZ5FZZFH+mdgVsx5i6YYhr1iEDaDq57DxOqBR+ZeAndcxOOA4f+uZoapOiXimqzo2n38dnK9++V71QGGMvjVnUSu++RavJOu+6UXTg5jCA21vk7dGjXr4M7jzn7f/98Q7vY1UUtFGDtMG+A1RAvifeeLGwflxnWFLl07QYbNLhVzSn5XzwMvqIZBd9gLpVrXx8kI0JYI00pjDvDu+dVKSw0JhOM9x10mHZLC1B7t0xDKgUT0HYhXVJVMKduTIX0dkk3mIOtGKcLijhdUY6w1mwh3MFt5/WHWR9aMj+g/Ib1d3D67EEmPVvMKsG+VLQ/JhGHL18L3+M6Jh43BaXWaFjuLiSRQlDIrUBrZpZjggKhQGVIo7ErelR50b1NC0tOkCfNE8XlghHMkcVgxPQBLB4WO1hqZEzjw9GuXG50GL1WOtta9rJaYz/wmDOss6VMbhM4I64x12CWynaokZWK7RE84X4AyF0aiy28aX4QC+EUq7M3XEtriHfu2wUEy9Gv/hdn6I3YNP8aQDdgy2thEM7PiCzKylAVFsNJ3Ph2Y+nMs2/6ZwEzFjsHwsw/q5ff//AXa/tetI6jptg3QbQ9n2ZxI2aHOm7whir0r41PTr/waABy4Vsfu/4nPc+LLc4drt95HkcmL++TbU/6A1PsOmfo/W8fL+3eqaLOeQL+0pxpomiJBdlYrdKrZ7yfC4KAQs/Rx3ev0ZUwP758jq7eX1z+12v06UqYVz+hp+vlBgnKzJIqRJZS+1FpUilKDHzrh1f/+/979iRIEWqWCWVcnx4gU88KHB7HoxNz3z2v+Y3jxasaqfAVzx8X0m3ZtAfzIxvGHfzAh/DtKaZb6+QzU6bCHL198z6I7J9S0HS+rOM44/9IQc/CtLXofjUiFDayX3jCETzGN3jHOSywoWv8ACPSgbuv0Zs8V+CndVweQqd5eklRHhvnPDUWcnX+7tq9SqPhsQLrCaMfHaeS01T9242uri0qI94vS8MjJ0FEoaFde5yGtSaWuela0wqIFro4z5n9MubbgG1rln/4nZuQAaxJCBdc+ht+0WWBASrbXOsket2hTxpG7z2G11KZRiQPhG4OATY4AGY2+yWvnpj2bj9MLOrHpN7WuzHCCxqyG6fy4nrswPLFWkvCrMrp/EYDHQdZuaywWNCzxnQiUszZolI0R7MNwKQih6yhsJwpj2w9MCgaHdGWg4vOE/Q74BF1/3YJV3QHgKKFNDTzmd3x84zikzYXOsOZS8VPALo0Kg3weQKWmCeoFuYprkOq/idlAqLiPKs9cenU8r4Fb/dx1l+t7Ux4AA320iypEtSgj5uSPkef6mfsLTjAfkTXtQNs8BL8Nqap1aN6JlAmRkzjGmnvF3+OMOdBZaLcfhES3LCCxLwVVfYNZMJIpA085kygT1ejAoVAgmwyeRVdZFugskww9s0CVlTHzui1YBOUuLgXMXYqOvjbE2DrRitknIpF9EmRgLNVPhJqoSMaqFN5MG8FYAQikE4wRxj9ItUaq3w4pxuhNwtI9lII2xt/B7l0M2rWlIqw6hm5a+J9Y9zSYN4O1TlkELSMh8yIwQ6Z8HmukJZQMGPFkh+xEd7iimMxRRz/AAdlnSDSclEONth1WW4jKStrwS7AgO2+PLEjlZRAF4JVvH5wh0XssTKMVBwrBP2iUY3E08u712/lQs7n4envlGRmSZMfbwfZj3ZBdxtbeF9avC26byqzpML4ZPFRtHUVs3PCYQk9bslx1D9pqkYRlpUhclpK+yXHEb6pCKFaj+AMncePa452XOIJ4IWsiruQaoMChQkD3KYQTh0caQ9HK5UgwKdLKey7YuVWSDlsfogGilJ3V6t4/ehG3k2MXNdSqBngjObNfrwfpqcPM4E0M1VAfiIoLqBeRHuoS6wRzmVpXxezpEwhuRbbI3OEM/hOClmM5NXCTA7NXIv6aZUIq9wzkVv5I5VuCIDRL4xT9MYjdjYgwyHOXtFszN3J0YTxZv8Pkq4wSoIbn7UQlwqhPQYIEbPe/QRCuHy9G1+vEZsS4wmhM5myeiCw+Rld4hWTFWiXRBalkgUbyVCkUyN3KfCMQxHZHJ3vxo2JVSN2EiLZx7CjdaIgAh0Mow6XOQLBwPoNfqlPt/XKbu/bKNttyywrYfrlbLE1+hzKwDNyjFl/kBYE7/GCCqoYqbcEBIFEv35qATNLeGpDs92QR/aM/HCmjRoPftZ7Oqbt1oPt6eXuPXn1wq2VcF9B07Qxwg0rqLZy3Wl7ipZ0NIjkTyFaU4i9BwGNB088BnUgax3Tu/vBWOvHw/b0Q6ajDTk9eGveYbxvh4O9wY63AuEAYfD17u7l3t2pSc/OXbQoe1P7Ty5aL9VpBMgeOd4IkK+XHX/cf2SxRhtMc2SHyUc1qQSJeccOkB+TsmPMvQ2YsVHqoQSt56eOXrlTmWVWULOUDxAlwR1PMnJo+K+NHjj0UlIyqddpR1Tng+TeX2sR2cGXiTwh/3X28/ffo6dvL95cP0MXTBsmFhXTS5pDKXwQFy4XMnlfoF2RMMiWnTs8/DHDF0cyxpRM7FXcVf9pTzWEQXNjwCMfbejzfa4LgbT/pu635fgDnEIxUyxCbdK3mWKYx+pO19vIB5yzSrsVkFRIs4JxrJx4smLT3iEC73q4vAruuWb5lJ1G2pnynywj1F7EXl/M7SVPV2fxRuy66xDW8JWGLf+vdxLBJwNe8I4b2irLyMOuTKlSJgYMQjZAaqkWWLA/d2RVi3SscCixj6B0m6dGyD1nKlhLmqjrzy92OXgtXIsv17uok9X8K8XcLAlWFJWK5rJgAgcL7lri6RobRoXRe9PjOZ5yt2/xg27WtX6kZSLGtVfniRVcJVYGmiFtt7pbrE7Y7MgLm0Mk6pzmVGFD8yxaUtkO/rDC55d6xSZ4dq3kiuVN8zD/PVyW3GuqA8bwzX/ss9bVacMKznaTLJ9ol82Svtef2YxsMzg8FDInV8xFz5d9xX2kBVyjdMYcCn5fzZPegc7U+lGrEnoR2KjTUUFjxRppI5WT+BZaQQ2G1Z7At87st56Ed1+wPOd0Oin3DtY7VM4Fjrcl946Sc/V4jGm2e+1Xa3UYEps6OvsclRzbI7Pvs1SICqI25ZiXH1IhJ7AnD8igU41t+avUBr3DZMnEiEmX40SS45s+rT8JyPQvFbXiw+pHrsmZPkNvc1yiz/APpx/lUri6038OH0+0xCtqNSdOsUJfKqo2CHoQ6lIKTWuNKlycavebwW+mkZe+Bx6xkBWru0AKt33Xl28cz3pLE6C6ZaAPvjnqoZjClKe0DrM+j9etpTtNjKxt6B9eppGqhAjasfp58/K4yLNrIzVSY+chZt7CTH8QGK2ZyOVaI11SwuaM2E+eh+oEfZ7s8ILY7Tl8tzk36Cl0hKWCbJ8hCF0+a1ELVQLe8bd0gckGfdLdxrdNBLboF9JGz661K0xgsI+89m1TC1CBWjVgMvsiDije9AEIVP93Kk2hnGdIvu620yvUY915nXod2DHsMMho/jdHbHaavN6xrfoMX+96r2XdJWx9vAvocDfTOOyagEH3bLYJme4YBicUbkixv/gZygZijgQcrXCDLed0zoT31YNwgq5+BS5Hmg4CdkcViiXCbeuA6al/sQVj47NNvXffS2mkN2XjwzYGk2UxcQv87apAcDSwjtrHkWTIy4yJeBPEot4Nu2UoKkz7eAaEVLtsB47FtdHelvcHpnYOsE779u3BusSq5in75+fbrayXbNBKHdnbYW1Zl/x+0PZM9Jklrq2FVJt0B/5XXWLxt70dY2pEul3Ua/U89DRZsvz1BUDfs7cHU4kGu6r7re/e1SgXZFQYJctjREcuq9nAuXAQj/s1rbVN95QjAI6uumPae3guixKLTXMf4drBOH1nr6yoss9QxsRchpUCrG9T1wjtkR89K7LGbE3TdkWff0mVI/BLxfkG/WeFOZszmqMLqHt2zsEgKms6y4iUt+yBgu6/0xly62/tZ8zHtPno3Wa34fCyMqByHznCdP9d/9As4afseHe088mfoY+b0m196zmwxHEnOH54is6zqM1ke2hbHJwjQj3Roba1fWSmcNU1ymUXO+dZLKWqvf0QYv7wduTIW71yIrNTTYsy7RyiHaSwK+/13NdoKikTaSJdpOw69jxQiU3YNUlEhnXMaH8LsPLl9JEhV4pHPOYW1Iin0hijWaVieUNaMDVVGV7Esym3oKM/T13QUdMfu6A91ycQLPTOUAGqVXzjxMKPxs2NordUtJcqE1ujcktMUUvYkbkfYVlQr174/z73KLzw/+HzmkJuf8ypCmfn+e08YPTcbaYdPAePa2vU2mA7uR+IZk0qJuZUqZG463Dfk+yrrfjvJX3QPTsBknVf4nnrGAJXCsLaMumVCiwxGftduri9ZbuPkEGs2n/6Bx0maI0P/GTlkqpp/BFWZ/cZT0/PYfTjM3QO64dRo8pM1CxlhM7nVPnhn7SThbmjOS9NGjpuEbJ14HbRJ7rVKXrnSbM/j/VK3r81Svi00Q37M+ytYbeJZMrVPy6RoAtpmDvAcon1yAQoTaZuK9Q6Srf4+HBBe9TJJkANElx6PFY3Tq/rb8IJKZotpqio6PY3aqYefhwdtGylCdO6iq50AmRIlkrnrTsthgIYUqWS+kAHh9KWnpd2cXQDweld0mmSDImmM7iPIj+9gdTO3Y9RS3oeh+T9pecOHMdFqNY8W6V80fshVe/IDiKTZ5b1cBW9TaNOBZjdUm9RJ2pu8M12XEn7QQLZ+hPSEK+TCl3dvPnHu2t0bd8p9JsYmb6yxTZRJfUx2H5cyzC2IIbIkpJbfZQT+TAhnLYHWWjoXNOvs2kRBmmgfgThVgru0HKpYoOmkA+g5Do8mq4go0YD4GywqSab8NnGcoU5yx0jBpDoC8LJulrvEoRAsVu60X2xHYnz6wTSyLCXxpQ6YzCDNgloOMoUBCH4EdwmthB15YtUzGz23CgiiyJpn7gD8XZ4eIdQuAR/zRTlfUsztotlzbHItH6ogbd2ZSfDf/e7rWu0gti6UuOslGyKtOoQwg4DBBgAUmFrAMhKlliIQeOM1O2m/KqAyEjMdqK2zc3D4mce/v72zXv/7r3oLd88KEaqvu8/es82pm+zleRVKgK8qec4Cz/nppmMXY/zrQQzGj11SOhn0K0DCnvribo98AiQDu6GV4mk2VuP6yfBjE8XOOsWHayogkyBecURkYLQ0lhD+cad4Uh7hfU6pfR1hLcGez1C2yJaSmWQtPT99d/fhFJwg2SPzXdSLaZPsOwXGHRcrDPsmp0EG8X8/fK366tr9A7fFUzkzVjv8LHavU2ehtkZojiyLb+Nwe52batRn8Ili9HTs12VYzafrmDzoYvw6y0nVzs6zjIvla8ufJdej8VODPl0h/LAvQLqHRf/7euGm8IckQ81ydi3G/wl1oR+oOxGP64arPgmqFu44t7nSFeBFHWs0V+1UVIs/jbjmNxypg3N//rC/+158ykTc0rCH82ZomvMg4oMnvHWbxAWOdISjbClogumjdpYy35KYVFis/TN+hscUB+HAZLglJoKTVcI7eq1iFStLuSNPtlgToVp5aTUeP9RCVZSdabV3b/0UeqzuTXPsKav0YyatuGf0zmuuMngErxGc8w7tcfju+/srpvM/07mFadw/0ustDXwPapa3SG90Vwu2i/2rguoKNYDi/9gqv9twMA9eGFXgxTCDQ7NDO5bciesvYWL2nCDONRVyAFxfgIGdWVwB2pwfYFNlooOAuZk3IMWimQWH1VxGgqcnkIQRQAdCxp1QO/HJPLRtDHZfz45BFOT0CTX5j406WASlyYdTPbTpJn7PFBMT8BhO7k537O65IxsIlPAAT1g7z6nLmN59mM/0nWSwHAtl1iOWmD3YJBi+d1ry8rMZCXyLJyh7TAIt+3ZdwAOoEteJLxjcexGBWa1xUPEjX47GA0mpiBIr+PTTkzS0eMQLCjHpaZ5ZtjI9cyHOv4eDDxI1AEZXHy8/9cJdyMENPyUU2sAZEmQcLAPx2UkCncCBgOIYTVS8hHmO06L7IALM1ywc9RJqw5B7uO2sXz2OGw3HHyzFx9dzRLjpKvZEXiRJVaYGKqYNoxE5JQOuUYWGXlOSaUoVIKUSt5t6iFZsRVgWAat6QzBMs2Tu1/xg0zLFK++BXzw0++wAD9yhgPJvc0j0690OQgJ55/uw92LSGAO+ylPbguVDuRxPFpBg/hUaUckDidNG6UE9GkjtZ9IdUJSPKYdQAzfakWylZpnC9Un/Gmm4wBocHVrqkVfPQQ0rAiSohyXXkedvAV5iKQKtm44zSiERAJoXAiNmrAfiIkKqjs91EaenplKYbpbuIfb7aQMejxPOI/7rO+j6Znr1hPZbq5D9Q74Yb4DaIQR2Xx3UA9YX8nKMLHImNAGi0Hh2Em6KkBGA8hhPOJ7cQ703lQlZ+I2M3exLUUHGJk71AW8CwuVCgt1EBZy9kdEC0nxbr18eOODuu4Tl+z1Rdh16+ItPGfcBIZwBBePb4f41Q8zP+oOUVGXp/vvmaFFKRVWmyw+BjtgB3GJu/r+9aDCJ+s/0ScvHAa7A4PBlPAIy3dghrWOYbuy07QNst/hU7vPo76pXYg7143/oLtowAH3LKpmt1+fcvNIs1DT4hOeLQcVdaGGuStQlH9SoBYW7kINLuzvXWRV2kM9gPB1wT9dUdWfuXbambseCgPAIybF1psVl+fbLqz9bA+dOoNt2U5BAYAesDoTOb2Lt24X3C7ui8p4eLH3GSklWUYNZliAB4QyXPgn43Ix5vI7Iapkzba9Dj96V0oVSD6cZvU8lzpLd9cs+HteuFaWxpKZTA2P/gTKtNI0lsygDvCwMGQFzTSRUd8+VlDUhTlGCUPvTBoyQBftw2hQ4wHOlYTYuBrKe5xLZN0AzuUAzQDWLqlig05Zpy7eA7r/NOIxZecQDsMgrqioETjAt1RxSmdYp3DsANwDcFhRlTMy0hbpBBw83H43/J04uCKmeKxY49CDG8QBnIxJPA/0QL+DxkXJaaaX+OXPr+Jh4MCiHtiwyoT5Gis6bOx7mt7koKIO1PB95K58IRwEuW9IzEFr/ywsBKnAoZqJk7wtFuRelcVKSm3woNXfsariEF542X5q+kmW0B6eDlXan8LMXXija8YMxJu9bzjIjiXWy4xLeVvFi+GB9LCAUQ/wOBpRX5IDYyTO35Tl1AxHshy/uIOKelDDR54ie+XwhJU5pXlkwlM/onb3wpKQSkXWHgHmQdpjzNRzHw+0ll7j4MGL527aQ7D+tBXT3dYs/Mv/HwAA///ei04u" } diff --git a/x-pack/filebeat/module/juniper/netscreen/test/generated.log-expected.json b/x-pack/filebeat/module/juniper/netscreen/test/generated.log-expected.json index fb4fca25df2..da17c3a5f76 100644 --- a/x-pack/filebeat/module/juniper/netscreen/test/generated.log-expected.json +++ b/x-pack/filebeat/module/juniper/netscreen/test/generated.log-expected.json @@ -2399,8 +2399,8 @@ "observer.type": "Firewall", "observer.vendor": "Juniper", "related.ip": [ - "10.119.181.171", - "10.166.144.66" + "10.166.144.66", + "10.119.181.171" ], "rsa.internal.messageid": "00625", "rsa.misc.hardware_id": "dol", diff --git a/x-pack/filebeat/module/juniper/srx/_meta/fields.yml b/x-pack/filebeat/module/juniper/srx/_meta/fields.yml new file mode 100644 index 00000000000..55ded3a11e6 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/_meta/fields.yml @@ -0,0 +1,488 @@ +- name: juniper.srx + type: group + release: beta + default_field: false + overwrite: true + description: > + Module for parsing junipersrx syslog. + fields: + - name: reason + type: keyword + description: > + reason + + - name: connection_tag + type: keyword + description: > + connection tag + + - name: service_name + type: keyword + description: > + service name + + - name: nat_connection_tag + type: keyword + description: > + nat connection tag + + - name: src_nat_rule_type + type: keyword + description: > + src nat rule type + + - name: src_nat_rule_name + type: keyword + description: > + src nat rule name + + - name: dst_nat_rule_type + type: keyword + description: > + dst nat rule type + + - name: dst_nat_rule_name + type: keyword + description: > + dst nat rule name + + - name: protocol_id + type: keyword + description: > + protocol id + + - name: policy_name + type: keyword + description: > + policy name + + - name: session_id_32 + type: keyword + description: > + session id 32 + + - name: session_id + type: keyword + description: > + session id + + - name: outbound_packets + type: integer + description: > + packets from client + + - name: outbound_bytes + type: integer + description: > + bytes from client + + - name: inbound_packets + type: integer + description: > + packets from server + + - name: inbound_bytes + type: integer + description: > + bytes from server + + - name: elapsed_time + type: date + description: > + elapsed time + + - name: application + type: keyword + description: > + application + + - name: nested_application + type: keyword + description: > + nested application + + - name: username + type: keyword + description: > + username + + - name: roles + type: keyword + description: > + roles + + - name: encrypted + type: keyword + description: > + encrypted + + - name: application_category + type: keyword + description: > + application category + + - name: application_sub_category + type: keyword + description: > + application sub category + + - name: application_characteristics + type: keyword + description: > + application characteristics + + - name: secure_web_proxy_session_type + type: keyword + description: > + secure web proxy session type + + - name: peer_session_id + type: keyword + description: > + peer session id + + - name: peer_source_address + type: ip + description: > + peer source address + + - name: peer_source_port + type: integer + description: > + peer source port + + - name: peer_destination_address + type: ip + description: > + peer destination address + + - name: peer_destination_port + type: integer + description: > + peer destination port + + - name: hostname + type: keyword + description: > + hostname + + - name: src_vrf_grp + type: keyword + description: > + src_vrf_grp + + - name: dst_vrf_grp + type: keyword + description: > + dst_vrf_grp + + - name: icmp_type + type: integer + description: > + icmp type + + - name: process + type: keyword + description: > + process that generated the message + + - name: apbr_rule_type + type: keyword + description: > + apbr rule type + + - name: dscp_value + type: integer + description: > + apbr rule type + + - name: logical_system_name + type: keyword + description: > + logical system name + + - name: profile_name + type: keyword + description: > + profile name + + - name: routing_instance + type: keyword + description: > + routing instance + + - name: rule_name + type: keyword + description: > + rule name + + - name: uplink_tx_bytes + type: integer + description: > + uplink tx bytes + + - name: uplink_rx_bytes + type: integer + description: > + uplink rx bytes + + - name: obj + type: keyword + description: > + url path + + - name: url + type: keyword + description: > + url domain + + - name: profile + type: keyword + description: > + filter profile + + - name: category + type: keyword + description: > + filter category + + - name: filename + type: keyword + description: > + filename + + - name: temporary_filename + type: keyword + description: > + temporary_filename + + - name: name + type: keyword + description: > + name + + - name: error_message + type: keyword + description: > + error_message + + - name: error_code + type: keyword + description: > + error_code + + - name: action + type: keyword + description: > + action + + - name: protocol + type: keyword + description: > + protocol + + - name: protocol_name + type: keyword + description: > + protocol name + + - name: type + type: keyword + description: > + type + + - name: repeat_count + type: integer + description: > + repeat count + + - name: alert + type: keyword + description: > + repeat alert + + - name: message_type + type: keyword + description: > + message type + + - name: threat_severity + type: keyword + description: > + threat severity + + - name: application_name + type: keyword + description: > + application name + + - name: attack_name + type: keyword + description: > + attack name + + - name: index + type: keyword + description: > + index + + - name: message + type: keyword + description: > + mesagge + + - name: epoch_time + type: date + description: > + epoch time + + - name: packet_log_id + type: integer + description: > + packet log id + + - name: export_id + type: integer + description: > + packet log id + + - name: ddos_application_name + type: keyword + description: > + ddos application name + + - name: connection_hit_rate + type: integer + description: > + connection hit rate + + - name: time_scope + type: keyword + description: > + time scope + + - name: context_hit_rate + type: integer + description: > + context hit rate + + - name: context_value_hit_rate + type: integer + description: > + context value hit rate + + - name: time_count + type: integer + description: > + time count + + - name: time_period + type: integer + description: > + time period + + - name: context_value + type: keyword + description: > + context value + + - name: context_name + type: keyword + description: > + context name + + - name: ruleebase_name + type: keyword + description: > + ruleebase name + + - name: verdict_source + type: keyword + description: > + verdict source + + - name: verdict_number + type: integer + description: > + verdict number + + - name: file_category + type: keyword + description: > + file category + + - name: sample_sha256 + type: keyword + description: > + sample sha256 + + - name: malware_info + type: keyword + description: > + malware info + + - name: client_ip + type: ip + description: > + client ip + + - name: tenant_id + type: keyword + description: > + tenant id + + - name: timestamp + type: date + description: > + timestamp + + - name: th + type: keyword + description: > + th + + - name: status + type: keyword + description: > + status + + - name: state + type: keyword + description: > + state + + - name: file_hash_lookup + type: keyword + description: > + file hash lookup + + - name: file_name + type: keyword + description: > + file name + + - name: action_detail + type: keyword + description: > + action detail + + - name: sub_category + type: keyword + description: > + sub category + + - name: feed_name + type: keyword + description: > + feed name + + - name: occur_count + type: integer + description: > + occur count + + - name: tag + type: keyword + description: > + system log message tag, which uniquely identifies the message. + diff --git a/x-pack/filebeat/module/juniper/srx/config/srx.yml b/x-pack/filebeat/module/juniper/srx/config/srx.yml new file mode 100644 index 00000000000..6af16945317 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/config/srx.yml @@ -0,0 +1,31 @@ +{{ if eq .input "tcp" }} + +type: tcp +host: "{{.syslog_host}}:{{.syslog_port}}" + +{{ else if eq .input "udp" }} + +type: udp +host: "{{.syslog_host}}:{{.syslog_port}}" + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} + +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - add_locale: ~ + - add_fields: + target: '' + fields: + ecs.version: 1.5.0 diff --git a/x-pack/filebeat/module/juniper/srx/ingest/atp.yml b/x-pack/filebeat/module/juniper/srx/ingest/atp.yml new file mode 100644 index 00000000000..b93e8da9f98 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/atp.yml @@ -0,0 +1,363 @@ +description: Pipeline for parsing junipersrx firewall logs (atp pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- set: + field: event.kind + value: alert + if: '["SRX_AAMW_ACTION_LOG", "AAMW_MALWARE_EVENT_LOG", "AAMW_HOST_INFECTED_EVENT_LOG", "AAMW_ACTION_LOG"].contains(ctx.juniper?.srx?.tag) && ctx.juniper?.srx?.action != "PERMIT"' +- append: + field: event.category + value: malware + if: '["SRX_AAMW_ACTION_LOG", "AAMW_MALWARE_EVENT_LOG", "AAMW_HOST_INFECTED_EVENT_LOG", "AAMW_ACTION_LOG"].contains(ctx.juniper?.srx?.tag) && ctx.juniper?.srx?.action != "PERMIT"' +- append: + field: event.type + value: + - info + - denied + - connection + if: "ctx.juniper?.srx?.action == 'BLOCK' || ctx.juniper?.srx?.tag == 'AAMW_MALWARE_EVENT_LOG'" +- append: + field: event.type + value: + - allowed + - connection + if: "ctx.juniper?.srx?.action != 'BLOCK' && ctx.juniper?.srx?.tag != 'AAMW_MALWARE_EVENT_LOG'" +- set: + field: event.action + value: malware_detected + if: "ctx.juniper?.srx?.action == 'BLOCK' || ctx.juniper?.srx?.tag == 'AAMW_MALWARE_EVENT_LOG'" + + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_port != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_server + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_server != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.packets_from_server + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_server != null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_client + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_client != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.packets_from_client + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_client != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" +- rename: + field: juniper.srx.hostname + target_field: source.domain + ignore_missing: true + if: "ctx.juniper?.srx?.hostname != null" +- rename: + field: juniper.srx.client_ip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.client_ip != null" + +###################### +## ECS URL Mapping ## +###################### +- rename: + field: juniper.srx.http_host + target_field: url.domain + ignore_missing: true + if: "ctx.juniper?.srx?.http_host != null" + +############################# +## ECS Network/Geo Mapping ## +############################# +- rename: + field: juniper.srx.protocol_id + target_field: network.iana_number + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_id != null" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +############### +## Timestamp ## +############### +- date: + if: 'ctx.juniper.srx?.timestamp != null' + field: juniper.srx.timestamp + target_field: juniper.srx.timestamp + formats: + - 'EEE MMM dd HH:mm:ss yyyy' + - 'EEE MMM d HH:mm:ss yyyy' + on_failure: + - remove: + field: + - juniper.srx.timestamp + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.bytes_from_client + - juniper.srx.packets_from_client + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.bytes_from_server + - juniper.srx.packets_from_server + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/flow.yml b/x-pack/filebeat/module/juniper/srx/ingest/flow.yml new file mode 100644 index 00000000000..1a488a57bd8 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/flow.yml @@ -0,0 +1,360 @@ +description: Pipeline for parsing junipersrx firewall logs (flow pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- rename: + field: juniper.srx.application_risk + target_field: event.risk_score + ignore_missing: true + if: "ctx.juniper?.srx?.application_risk != null" +- append: + field: event.type + value: + - start + - allowed + - connection + if: "ctx.juniper?.srx?.tag.endsWith('CREATE') || ctx.juniper?.srx?.tag.endsWith('UPDATE') || ctx.juniper?.srx?.tag.endsWith('CREATE_LS') || ctx.juniper?.srx?.tag.endsWith('UPDATE_LS')" +- append: + field: event.type + value: + - end + - allowed + - connection + if: "ctx.juniper?.srx?.tag.endsWith('CLOSE') || ctx.juniper?.srx?.tag.endsWith('CLOSE_LS')" +- append: + field: event.type + value: + - denied + - connection + if: "ctx.juniper?.srx?.tag.endsWith('DENY') || ctx.juniper?.srx?.tag.endsWith('DENY_LS')" +- set: + field: event.action + value: flow_started + if: "ctx.juniper?.srx?.tag.endsWith('CREATE') || ctx.juniper?.srx?.tag.endsWith('UPDATE') || ctx.juniper?.srx?.tag.endsWith('CREATE_LS') || ctx.juniper?.srx?.tag.endsWith('UPDATE_LS')" +- set: + field: event.action + value: flow_close + if: "ctx.juniper?.srx?.tag.endsWith('CLOSE') || ctx.juniper?.srx?.tag.endsWith('CLOSE_LS')" +- set: + field: event.action + value: flow_deny + if: "ctx.juniper?.srx?.tag.endsWith('DENY') || ctx.juniper?.srx?.tag.endsWith('DENY_LS')" + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx?.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_port != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_server + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_server != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.packets_from_server + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_server != null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_client + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_client != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.packets_from_client + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_client != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" + +###################### +## ECS Rule Mapping ## +###################### +- rename: + field: juniper.srx.policy_name + target_field: rule.name + ignore_missing: true + if: "ctx.juniper?.srx?.policy_name != null" + +############################# +## ECS Network/Geo Mapping ## +############################# +- rename: + field: juniper.srx.protocol_id + target_field: network.iana_number + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_id != null" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true +- script: + lang: painless + source: "ctx.network.bytes = ctx.source.bytes + ctx.destination.bytes" + if: "ctx?.source?.bytes != null && ctx?.destination?.bytes != null" + ignore_failure: true +- script: + lang: painless + source: "ctx.network.packets = ctx.client.packets + ctx.server.packets" + if: "ctx?.client?.packets != null && ctx?.server?.packets != null" + ignore_failure: true + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.bytes_from_client + - juniper.srx.packets_from_client + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.bytes_from_server + - juniper.srx.packets_from_server + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/idp.yml b/x-pack/filebeat/module/juniper/srx/ingest/idp.yml new file mode 100644 index 00000000000..808185410d7 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/idp.yml @@ -0,0 +1,287 @@ +description: Pipeline for parsing junipersrx firewall logs (idp pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- set: + field: event.kind + value: alert + if: '["IDP_ATTACK_LOG_EVENT", "IDP_APPDDOS_APP_STATE_EVENT", "IDP_APPDDOS_APP_ATTACK_EVENT", "IDP_ATTACK_LOG_EVENT_LS", "IDP_APPDDOS_APP_STATE_EVENT_LS", "IDP_APPDDOS_APP_ATTACK_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.category + value: intrusion_detection + if: '["IDP_ATTACK_LOG_EVENT", "IDP_APPDDOS_APP_STATE_EVENT", "IDP_APPDDOS_APP_ATTACK_EVENT", "IDP_ATTACK_LOG_EVENT_LS", "IDP_APPDDOS_APP_STATE_EVENT_LS", "IDP_APPDDOS_APP_ATTACK_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - info + - denied + - connection + if: '["IDP_ATTACK_LOG_EVENT", "IDP_APPDDOS_APP_STATE_EVENT", "IDP_APPDDOS_APP_ATTACK_EVENT", "IDP_ATTACK_LOG_EVENT_LS", "IDP_APPDDOS_APP_STATE_EVENT_LS", "IDP_APPDDOS_APP_ATTACK_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - allowed + - connection + if: '!["IDP_ATTACK_LOG_EVENT", "IDP_APPDDOS_APP_STATE_EVENT", "IDP_APPDDOS_APP_ATTACK_EVENT", "IDP_ATTACK_LOG_EVENT_LS", "IDP_APPDDOS_APP_STATE_EVENT_LS", "IDP_APPDDOS_APP_ATTACK_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: application_ddos + if: '["IDP_APPDDOS_APP_STATE_EVENT", "IDP_APPDDOS_APP_ATTACK_EVENT", "IDP_APPDDOS_APP_STATE_EVENT_LS", "IDP_APPDDOS_APP_ATTACK_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: security_threat + if: '["IDP_ATTACK_LOG_EVENT", "IDP_ATTACK_LOG_EVENT_LS"].contains(ctx.juniper?.srx?.tag)' + + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx['nat_destination_port'] != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.inbound_bytes + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.inbound_bytes != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.inbound_packets + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.inbound_packets !=null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.outbound_bytes + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.outbound_bytes != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.outbound_packets + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.outbound_packets != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" + +###################### +## ECS Rule Mapping ## +###################### +- rename: + field: juniper.srx.rulebase_name + target_field: rule.name + ignore_missing: true + if: "ctx.juniper?.srx?.rulebase_name != null" +- rename: + field: juniper.srx.rule_name + target_field: rule.id + ignore_missing: true + if: "ctx.juniper?.srx?.rule_name != null" + +######################### +## ECS Network Mapping ## +######################### +- rename: + field: juniper.srx.protocol_name + target_field: network.protocol + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_name != null" + +######################### +## ECS message Mapping ## +######################### +- rename: + field: juniper.srx.message + target_field: message + ignore_missing: true + if: "ctx.juniper?.srx?.message != null" + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.outbound_bytes + - juniper.srx.outbound_packets + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.inbound_bytes + - juniper.srx.inbound_packets + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/ids.yml b/x-pack/filebeat/module/juniper/srx/ingest/ids.yml new file mode 100644 index 00000000000..039fdd64ccb --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/ids.yml @@ -0,0 +1,363 @@ +description: Pipeline for parsing junipersrx firewall logs (ids pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- set: + field: event.kind + value: alert + if: '["RT_SCREEN_TCP", "RT_SCREEN_UDP", "RT_SCREEN_ICMP", "RT_SCREEN_IP", "RT_SCREEN_TCP_DST_IP", "RT_SCREEN_TCP_SRC_IP", "RT_SCREEN_TCP_LS", "RT_SCREEN_UDP_LS", "RT_SCREEN_ICMP_LS", "RT_SCREEN_IP_LS", "RT_SCREEN_TCP_DST_IP_LS", "RT_SCREEN_TCP_SRC_IP_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.category + value: intrusion_detection + if: '["RT_SCREEN_TCP", "RT_SCREEN_UDP", "RT_SCREEN_ICMP", "RT_SCREEN_IP", "RT_SCREEN_TCP_DST_IP", "RT_SCREEN_TCP_SRC_IP", "RT_SCREEN_TCP_LS", "RT_SCREEN_UDP_LS", "RT_SCREEN_ICMP_LS", "RT_SCREEN_IP_LS", "RT_SCREEN_TCP_DST_IP_LS", "RT_SCREEN_TCP_SRC_IP_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - info + - denied + - connection + if: '["RT_SCREEN_TCP", "RT_SCREEN_UDP", "RT_SCREEN_ICMP", "RT_SCREEN_IP", "RT_SCREEN_TCP_DST_IP", "RT_SCREEN_TCP_SRC_IP", "RT_SCREEN_TCP_LS", "RT_SCREEN_UDP_LS", "RT_SCREEN_ICMP_LS", "RT_SCREEN_IP_LS", "RT_SCREEN_TCP_DST_IP_LS", "RT_SCREEN_TCP_SRC_IP_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - allowed + - connection + if: '!["RT_SCREEN_TCP", "RT_SCREEN_UDP", "RT_SCREEN_ICMP", "RT_SCREEN_IP", "RT_SCREEN_TCP_DST_IP", "RT_SCREEN_TCP_SRC_IP", "RT_SCREEN_TCP_LS", "RT_SCREEN_UDP_LS", "RT_SCREEN_ICMP_LS", "RT_SCREEN_IP_LS", "RT_SCREEN_TCP_DST_IP_LS", "RT_SCREEN_TCP_SRC_IP_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: flood_detected + if: '["ICMP flood!", "UDP flood!", "SYN flood!", "SYN flood Src-IP based!", "SYN flood Dst-IP based!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: scan_detected + if: "ctx.juniper?.srx?.attack_name == 'TCP port scan!'" +- set: + field: event.action + value: sweep_detected + if: '["TCP sweep!", "IP sweep!", "UDP sweep!", "Address sweep!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: fragment_detected + if: '["ICMP fragment!", "SYN fragment!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: spoofing_detected + if: "ctx.juniper?.srx?.attack_name == 'IP spoofing!'" +- set: + field: event.action + value: session_limit_detected + if: '["Src IP session limit!", "Dst IP session limit!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: attack_detected + if: '["Land attack!", "WinNuke attack!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: illegal_tcp_flag_detected + if: '["No TCP flag!", "SYN and FIN bits!", "FIN but no ACK bit!"].contains(ctx.juniper?.srx?.attack_name)' +- set: + field: event.action + value: tunneling_screen + if: "ctx.juniper?.srx?.attack_name.startsWith('Tunnel')" + + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_port != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_server + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_server != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.packets_from_server + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_server !=null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_client + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_client != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.packets_from_client + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_client != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" + +############################# +## ECS Network/Geo Mapping ## +############################# +- rename: + field: juniper.srx.protocol_id + target_field: network.iana_number + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_id != null" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.bytes_from_client + - juniper.srx.packets_from_client + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.bytes_from_server + - juniper.srx.packets_from_server + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml new file mode 100644 index 00000000000..5bc4d45e82e --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/pipeline.yml @@ -0,0 +1,275 @@ +# This module only supports syslog messages in the format "structured-data + brief" +# https://www.juniper.net/documentation/en_US/junos/topics/reference/configuration-statement/structured-data-edit-system.html +description: Pipeline for parsing junipersrx firewall logs +processors: +- grok: + field: message + patterns: + - '^<%{POSINT:syslog_pri}>(\d{1,3}\s)?(?:%{TIMESTAMP_ISO8601:_temp_.raw_date})\s%{SYSLOGHOST:syslog_hostname}\s%{PROG:syslog_program}\s(?:%{POSINT:syslog_pid}|-)?\s%{WORD:log_type}\s\[.+?\s%{GREEDYDATA:log.original}\]$' + +# split Juniper-SRX fields +- kv: + field: log.original + field_split: " (?=[a-z0-9\\_\\-]+=)" + value_split: "=" + prefix: "juniper.srx." + ignore_missing: true + ignore_failure: false + trim_value: "\"" + +# Converts all kebab-case key names to snake_case +- script: + lang: painless + source: >- + ctx.juniper.srx = ctx?.juniper?.srx.entrySet().stream().collect(Collectors.toMap(e -> e.getKey().replace('-', '_'), e -> e.getValue())); + +# +# Parse the date +# +- date: + if: "ctx.event.timezone == null" + field: _temp_.raw_date + target_field: "@timestamp" + formats: + - yyyy-MM-dd HH:mm:ss + - yyyy-MM-dd HH:mm:ss z + - yyyy-MM-dd HH:mm:ss Z + - ISO8601 +- date: + if: "ctx.event.timezone != null" + timezone: "{{ event.timezone }}" + field: _temp_.raw_date + target_field: "@timestamp" + formats: + - yyyy-MM-dd HH:mm:ss + - yyyy-MM-dd HH:mm:ss z + - yyyy-MM-dd HH:mm:ss Z + - ISO8601 + +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' + +# Can possibly be omitted if there is a solution for the equal signs and the calculation of the start time. +# -> juniper.srx.elapsed_time +- rename: + field: juniper.srx.elapsed_time + target_field: juniper.srx.duration + if: "ctx.juniper?.srx?.elapsed_time != null" + +# Sets starts, end and duration when start and duration is known +- script: + lang: painless + if: ctx?.juniper?.srx?.duration != null + source: >- + ctx.event.duration = Integer.parseInt(ctx.juniper.srx.duration) * 1000000000L; + ctx.event.start = ctx['@timestamp']; + ZonedDateTime start = ZonedDateTime.parse(ctx.event.start); + ctx.event.end = start.plus(ctx.event.duration, ChronoUnit.NANOS); + +# Removes all empty fields +- script: + lang: painless + params: + values: + - "None" + - "UNKNOWN" + - "N/A" + - "-" + source: >- + ctx?.juniper?.srx.entrySet().removeIf(entry -> params.values.contains(entry.getValue())); + +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.module + value: juniper +- set: + field: event.dataset + value: juniper.srx +- set: + field: event.severity + value: '{{syslog_pri}}' +- rename: + field: log.original + target_field: event.original + ignore_missing: true + +##################### +## ECS Log Mapping ## +##################### +# https://www.juniper.net/documentation/en_US/junos/topics/reference/general/syslog-interpreting-msg-generated-structured-data-format.html#fac_sev_codes +- set: + field: "log.level" + if: '["0", "8", "16", "24", "32", "40", "48", "56", "64", "72", "80", "88", "96", "104", "112", "128", "136", "144", "152", "160", "168", "176", "184"].contains(ctx.syslog_pri)' + value: emergency +- set: + field: "log.level" + if: '["1", "9", "17", "25", "33", "41", "49", "57", "65", "73", "81", "89", "97", "105", "113", "129", "137", "145", "153", "161", "169", "177", "185"].contains(ctx.syslog_pri)' + value: alert +- set: + field: "log.level" + if: '["2", "10", "18", "26", "34", "42", "50", "58", "66", "74", "82", "90", "98", "106", "114", "130", "138", "146", "154", "162", "170", "178", "186"].contains(ctx.syslog_pri)' + value: critical +- set: + field: "log.level" + if: '["3", "11", "19", "27", "35", "43", "51", "59", "67", "75", "83", "91", "99", "107", "115", "131", "139", "147", "155", "163", "171", "179", "187"].contains(ctx.syslog_pri)' + value: error +- set: + field: "log.level" + if: '["4", "12", "20", "28", "36", "44", "52", "60", "68", "76", "84", "92", "100", "108", "116", "132", "140", "148", "156", "164", "172", "180", "188"].contains(ctx.syslog_pri)' + value: warning +- set: + field: "log.level" + if: '["5", "13", "21", "29", "37", "45", "53", "61", "69", "77", "85", "93", "101", "109", "117", "133", "141", "149", "157", "165", "173", "181", "189"].contains(ctx.syslog_pri)' + value: notification +- set: + field: "log.level" + if: '["6", "14", "22", "30", "38", "46", "54", "62", "70", "78", "86", "94", "102", "110", "118", "134", "142", "150", "158", "166", "174", "182", "190"].contains(ctx.syslog_pri)' + value: informational +- set: + field: "log.level" + if: '["7", "15", "23", "31", "39", "47", "55", "63", "71", "79", "87", "95", "103", "111", "119", "135", "143", "151", "159", "167", "175", "183", "191"].contains(ctx.syslog_pri)' + value: debug + +########################## +## ECS Observer Mapping ## +########################## +- set: + field: observer.vendor + value: Juniper +- set: + field: observer.product + value: SRX +- set: + field: observer.type + value: firewall +- rename: + field: syslog_hostname + target_field: observer.name + ignore_missing: true +- rename: + field: juniper.srx.packet_incoming_interface + target_field: observer.ingress.interface.name + ignore_missing: true +- rename: + field: juniper.srx.destination_interface_name + target_field: observer.egress.interface.name + ignore_missing: true +- rename: + field: juniper.srx.source_interface_name + target_field: observer.ingress.interface.name + ignore_missing: true +- rename: + field: juniper.srx.interface_name + target_field: observer.ingress.interface.name + ignore_missing: true +- rename: + field: juniper.srx.source_zone_name + target_field: observer.ingress.zone + ignore_missing: true +- rename: + field: juniper.srx.source_zone + target_field: observer.ingress.zone + ignore_missing: true +- rename: + field: juniper.srx.destination_zone_name + target_field: observer.egress.zone + ignore_missing: true +- rename: + field: juniper.srx.destination_zone + target_field: observer.egress.zone + ignore_missing: true +- rename: + field: syslog_program + target_field: juniper.srx.process + ignore_missing: true +- rename: + field: log_type + target_field: juniper.srx.tag + ignore_missing: true + + +############# +## Cleanup ## +############# +- remove: + field: + - message + - _temp_ + - _temp + - juniper.srx.duration + - juniper.srx.dir_disp + - juniper.srx.srczone + - juniper.srx.dstzone + - juniper.srx.duration + - syslog_pri + ignore_missing: true + +################################ +## Product Specific Pipelines ## +################################ +- pipeline: + name: '{< IngestPipeline "flow" >}' + if: "ctx.juniper?.srx?.process == 'RT_FLOW'" +- pipeline: + name: '{< IngestPipeline "utm" >}' + if: "ctx.juniper?.srx?.process == 'RT_UTM'" +- pipeline: + name: '{< IngestPipeline "idp" >}' + if: "ctx.juniper?.srx?.process == 'RT_IDP'" +- pipeline: + name: '{< IngestPipeline "ids" >}' + if: "ctx.juniper?.srx?.process == 'RT_IDS'" +- pipeline: + name: '{< IngestPipeline "atp" >}' + if: "ctx.juniper?.srx?.process == 'RT_AAMW'" +- pipeline: + name: '{< IngestPipeline "secintel" >}' + if: "ctx.juniper?.srx?.process == 'RT_SECINTEL'" + +######################### +## ECS Related Mapping ## +######################### +- append: + if: 'ctx.source?.ip != null' + field: related.ip + value: '{{source.ip}}' + ignore_failure: true +- append: + if: 'ctx.destination?.ip != null' + field: related.ip + value: '{{destination.ip}}' + ignore_failure: true +- append: + if: 'ctx.source?.nat?.ip != null' + field: related.ip + value: '{{source.nat.ip}}' + ignore_failure: true +- append: + if: 'ctx?.destination?.nat?.ip != null' + field: related.ip + value: '{{destination.nat.ip}}' + ignore_failure: true + +- append: + if: 'ctx.url?.domain != null' + field: related.hosts + value: '{{url.domain}}' + ignore_failure: true +- append: + if: 'ctx.source?.domain != null' + field: related.hosts + value: '{{source.domain}}' + ignore_failure: true +- append: + if: 'ctx.destination?.domain != null' + field: related.hosts + value: '{{destination.domain}}' + ignore_failure: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/secintel.yml b/x-pack/filebeat/module/juniper/srx/ingest/secintel.yml new file mode 100644 index 00000000000..f2abb2bcf9c --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/secintel.yml @@ -0,0 +1,349 @@ +description: Pipeline for parsing junipersrx firewall logs (secintel pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- set: + field: event.kind + value: alert + if: 'ctx.juniper?.srx?.tag == "SECINTEL_ACTION_LOG" && ctx.juniper?.srx?.action != "PERMIT"' +- append: + field: event.category + value: malware + if: 'ctx.juniper?.srx?.tag == "SECINTEL_ACTION_LOG" && ctx.juniper?.srx?.action != "PERMIT"' +- append: + field: event.type + value: + - info + - denied + - connection + if: "ctx.juniper?.srx?.action == 'BLOCK'" +- append: + field: event.type + value: + - allowed + - connection + if: "ctx.juniper?.srx?.action != 'BLOCK'" +- set: + field: event.action + value: malware_detected + if: "ctx.juniper?.srx?.action == 'BLOCK'" + + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_port != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_server + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_server != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.packets_from_server + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_server !=null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_client + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_client != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.packets_from_client + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_client != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" +- rename: + field: juniper.srx.hostname + target_field: source.address + ignore_missing: true + if: "ctx.juniper?.srx?.hostname != null" +- rename: + field: juniper.srx.client_ip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.client_ip != null" + +###################### +## ECS URL Mapping ## +###################### +- rename: + field: juniper.srx.http_host + target_field: url.domain + ignore_missing: true + if: "ctx.juniper?.srx?.http_host != null" + +############################# +## ECS Network/Geo Mapping ## +############################# +- rename: + field: juniper.srx.protocol_id + target_field: network.iana_number + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_id != null" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.bytes_from_client + - juniper.srx.packets_from_client + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.bytes_from_server + - juniper.srx.packets_from_server + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/ingest/utm.yml b/x-pack/filebeat/module/juniper/srx/ingest/utm.yml new file mode 100644 index 00000000000..a80e5a94d97 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/ingest/utm.yml @@ -0,0 +1,388 @@ +description: Pipeline for parsing junipersrx firewall logs (utm pipeline) +processors: +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: event +- set: + field: event.outcome + value: success + if: "ctx.juniper?.srx?.tag != null" +- append: + field: event.category + value: network +- rename: + field: juniper.srx.urlcategory_risk + target_field: event.risk_score + ignore_missing: true + if: "ctx.juniper?.srx?.urlcategory_risk != null" +- set: + field: event.kind + value: alert + if: '["AV_VIRUS_DETECTED_MT", "WEBFILTER_URL_BLOCKED", "ANTISPAM_SPAM_DETECTED_MT", "CONTENT_FILTERING_BLOCKED_MT", "AV_VIRUS_DETECTED_MT_LS", "WEBFILTER_URL_BLOCKED_LS", "ANTISPAM_SPAM_DETECTED_MT_LS", "CONTENT_FILTERING_BLOCKED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.category + value: malware + if: '["AV_VIRUS_DETECTED_MT", "WEBFILTER_URL_BLOCKED", "ANTISPAM_SPAM_DETECTED_MT", "CONTENT_FILTERING_BLOCKED_MT", "AV_VIRUS_DETECTED_MT_LS", "WEBFILTER_URL_BLOCKED_LS", "ANTISPAM_SPAM_DETECTED_MT_LS", "CONTENT_FILTERING_BLOCKED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - info + - denied + - connection + if: '["AV_VIRUS_DETECTED_MT", "WEBFILTER_URL_BLOCKED", "ANTISPAM_SPAM_DETECTED_MT", "CONTENT_FILTERING_BLOCKED_MT", "AV_VIRUS_DETECTED_MT_LS", "WEBFILTER_URL_BLOCKED_LS", "ANTISPAM_SPAM_DETECTED_MT_LS", "CONTENT_FILTERING_BLOCKED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- append: + field: event.type + value: + - allowed + - connection + if: '!["AV_VIRUS_DETECTED_MT", "WEBFILTER_URL_BLOCKED", "ANTISPAM_SPAM_DETECTED_MT", "CONTENT_FILTERING_BLOCKED_MT", "AV_VIRUS_DETECTED_MT_LS", "WEBFILTER_URL_BLOCKED_LS", "ANTISPAM_SPAM_DETECTED_MT_LS", "CONTENT_FILTERING_BLOCKED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: web_filter + if: '["WEBFILTER_URL_BLOCKED", "WEBFILTER_URL_BLOCKED_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: content_filter + if: '["CONTENT_FILTERING_BLOCKED_MT", "CONTENT_FILTERING_BLOCKED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: antispam_filter + if: '["ANTISPAM_SPAM_DETECTED_MT", "ANTISPAM_SPAM_DETECTED_MT_LS"].contains(ctx.juniper?.srx?.tag)' +- set: + field: event.action + value: virus_detected + if: '["AV_VIRUS_DETECTED_MT", "AV_VIRUS_DETECTED_MT_LS"].contains(ctx.juniper?.srx?.tag)' + + +#################################### +## ECS Server/Destination Mapping ## +#################################### +- rename: + field: juniper.srx.destination_address + target_field: destination.ip + ignore_missing: true + if: "ctx.juniper?.srx?.destination_address != null" +- set: + field: server.ip + value: '{{destination.ip}}' + if: "ctx.destination?.ip != null" +- rename: + field: juniper.srx.nat_destination_address + target_field: destination.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_address != null" +- convert: + field: juniper.srx.destination_port + target_field: destination.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.destination_port != null" +- set: + field: server.port + value: '{{destination.port}}' + if: "ctx.destination?.port != null" +- convert: + field: server.port + target_field: server.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.port != null" +- convert: + field: juniper.srx.nat_destination_port + target_field: destination.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_destination_port != null" +- set: + field: server.nat.port + value: '{{destination.nat.port}}' + if: "ctx.destination?.nat?.port != null" +- convert: + field: server.nat.port + target_field: server.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_server + target_field: destination.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_server != null" +- set: + field: server.bytes + value: '{{destination.bytes}}' + if: "ctx.destination?.bytes != null" +- convert: + field: server.bytes + target_field: server.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.bytes != null" +- convert: + field: juniper.srx.packets_from_server + target_field: destination.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_server !=null" +- set: + field: server.packets + value: '{{destination.packets}}' + if: "ctx.destination?.packets != null" +- convert: + field: server.packets + target_field: server.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.server?.packets != null" + +############################### +## ECS Client/Source Mapping ## +############################### +- rename: + field: juniper.srx.source_address + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.source_address != null" +- set: + field: client.ip + value: '{{source.ip}}' + if: "ctx.source?.ip != null" +- rename: + field: juniper.srx.nat_source_address + target_field: source.nat.ip + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_address != null" +- rename: + field: juniper.srx.sourceip + target_field: source.ip + ignore_missing: true + if: "ctx.juniper?.srx?.sourceip != null" +- convert: + field: juniper.srx.source_port + target_field: source.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.source_port != null" +- set: + field: client.port + value: '{{source.port}}' + if: "ctx.source?.port != null" +- convert: + field: client.port + target_field: client.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.port != null" +- convert: + field: juniper.srx.nat_source_port + target_field: source.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.nat_source_port != null" +- set: + field: client.nat.port + value: '{{source.nat.port}}' + if: "ctx.source?.nat?.port != null" +- convert: + field: client.nat.port + target_field: client.nat.port + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.nat?.port != null" +- convert: + field: juniper.srx.bytes_from_client + target_field: source.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.bytes_from_client != null" +- set: + field: client.bytes + value: '{{source.bytes}}' + if: "ctx.source?.bytes != null" +- convert: + field: client.bytes + target_field: client.bytes + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.bytes != null" +- convert: + field: juniper.srx.packets_from_client + target_field: source.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.juniper?.srx?.packets_from_client != null" +- set: + field: client.packets + value: '{{source.packets}}' + if: "ctx.source?.packets != null" +- convert: + field: client.packets + target_field: client.packets + type: long + ignore_failure: true + ignore_missing: true + if: "ctx.client?.packets != null" +- rename: + field: juniper.srx.username + target_field: source.user.name + ignore_missing: true + if: "ctx.juniper?.srx?.username != null" + +###################### +## ECS Rule Mapping ## +###################### +- rename: + field: juniper.srx.policy_name + target_field: rule.name + ignore_missing: true + if: "ctx.juniper?.srx?.policy_name != null" + +##################### +## ECS URL Mapping ## +##################### +- rename: + field: juniper.srx.url + target_field: url.domain + ignore_missing: true + if: "ctx.juniper?.srx?.url != null" +- rename: + field: juniper.srx.obj + target_field: url.path + ignore_missing: true + if: "ctx.juniper?.srx?.obj != null" + +###################### +## ECS File Mapping ## +###################### +- rename: + field: juniper.srx.filename + target_field: file.name + ignore_missing: true + if: "ctx.juniper?.srx?.filename != null" + +######################### +## ECS Network Mapping ## +######################### +- rename: + field: juniper.srx.protocol + target_field: network.protocol + ignore_missing: true + if: "ctx.juniper?.srx?.protocol != null" + +############################# +## ECS Network/Geo Mapping ## +############################# +- rename: + field: juniper.srx.protocol_id + target_field: network.iana_number + ignore_missing: true + if: "ctx.juniper?.srx?.protocol_id != null" +- geoip: + field: source.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true +- geoip: + field: source.nat.ip + target_field: source.geo + ignore_missing: true + if: "ctx.source?.geo == null" +- geoip: + field: destination.nat.ip + target_field: destination.geo + ignore_missing: true + if: "ctx.destination?.geo == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: source.nat.ip + target_field: source.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.source?.as == null" +- geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.nat.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + if: "ctx.destination?.as == null" +- rename: + field: source.as.asn + target_field: source.as.number + ignore_missing: true +- rename: + field: source.as.organization_name + target_field: source.as.organization.name + ignore_missing: true +- rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true +- rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + +############# +## Cleanup ## +############# +- remove: + field: + - juniper.srx.destination_port + - juniper.srx.nat_destination_port + - juniper.srx.bytes_from_client + - juniper.srx.packets_from_client + - juniper.srx.source_port + - juniper.srx.nat_source_port + - juniper.srx.bytes_from_server + - juniper.srx.packets_from_server + ignore_missing: true + +on_failure: +- set: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/juniper/srx/manifest.yml b/x-pack/filebeat/module/juniper/srx/manifest.yml new file mode 100644 index 00000000000..879be66b99d --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/manifest.yml @@ -0,0 +1,26 @@ +module_version: 1.0 + +var: + - name: syslog_host + default: localhost + - name: tags + default: ["juniper.srx", "forwarded"] + - name: syslog_port + default: 9006 + - name: input + default: udp + +ingest_pipeline: + - ingest/pipeline.yml + - ingest/flow.yml + - ingest/utm.yml + - ingest/idp.yml + - ingest/ids.yml + - ingest/atp.yml + - ingest/secintel.yml + +input: config/srx.yml + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/juniper/srx/test/atp.log b/x-pack/filebeat/module/juniper/srx/test/atp.log new file mode 100644 index 00000000000..95c8210f038 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/atp.log @@ -0,0 +1,4 @@ +<14>1 2013-12-14T16:06:59.134Z pinarello RT_AAMW - SRX_AAMW_ACTION_LOG [junos@xxx.x.x.x.x.28 http-host="www.mytest.com" file-category="executable" action="BLOCK" verdict-number="8" verdict-source=”cloud/blacklist/whitelist” source-address="10.10.10.1" source-port="57116" destination-address="187.19.188.200" destination-port="80" protocol-id="6" application="UNKNOWN" nested-application="UNKNOWN" policy-name="argon_policy" username="user1" session-id-32="50000002" source-zone-name="untrust" destination-zone-name="trust"] +<14>1 2016-09-20T10:43:30.330-07:00 host-example RT_AAMW - AAMW_MALWARE_EVENT_LOG [junos@xxxx.1.1.x.x.xxx timestamp="Thu Jun 23 09:55:38 2016" tenant-id="ABC123456" sample-sha256="ABC123" client-ip="192.0.2.0" verdict-number="9" malware-info="Eicar:TestVirus" username="admin" hostname="host.example.com"] +<11>1 2016-09-20T10:40:30.050-07:00 host-example RT_AAMW - AAMW_HOST_INFECTED_EVENT_LOG [junos@xxxx.1.1.x.x.xxx timestamp="Thu Jun 23 09:55:38 2016" tenant-id="ABC123456" client-ip="192.0.2.0" hostname="host.example.com" status="in_progress" policy-name="default" th="7" state="added" reason="malware" message="malware analysis detected host downloaded a malicious_file with score 9, sha256 ABC123"] +<165>1 2007-02-15T09:17:15.719Z aamw1 RT_AAMW - AAMW_ACTION_LOG [junos@2636.1.1.1.2.129 hostname="dummy_host" file-category="executable" verdict-number="10" malware-info="Testfile" action="PERMIT" list-hit="N/A" file-hash-lookup="FALSE" source-address="1.1.1.1" source-port="60148" destination-address="10.0.0.1" destination-port="80" protocol-id="6" application="HTTP" nested-application="N/A" policy-name="test-policy" username="N/A" roles="N/A" session-id-32="502156" source-zone-name="Inside" destination-zone-name="Outside" sample-sha256="e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494" file-name="dummy_file" url="dummy_url"] diff --git a/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json new file mode 100644 index 00000000000..4187866594e --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/atp.log-expected.json @@ -0,0 +1,240 @@ +[ + { + "@timestamp": "2013-12-14T14:06:59.134-02:00", + "client.ip": "10.10.10.1", + "client.port": 57116, + "destination.as.number": 28126, + "destination.as.organization.name": "BRISANET SERVICOS DE TELECOMUNICACOES LTDA", + "destination.geo.city_name": "Juazeiro do Norte", + "destination.geo.continent_name": "South America", + "destination.geo.country_iso_code": "BR", + "destination.geo.country_name": "Brazil", + "destination.geo.location.lat": -7.1467, + "destination.geo.location.lon": -39.247, + "destination.geo.region_iso_code": "BR-CE", + "destination.geo.region_name": "Ceara", + "destination.ip": "187.19.188.200", + "destination.port": 80, + "event.action": "malware_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "http-host=\"www.mytest.com\" file-category=\"executable\" action=\"BLOCK\" verdict-number=\"8\" verdict-source=\u201dcloud/blacklist/whitelist\u201d source-address=\"10.10.10.1\" source-port=\"57116\" destination-address=\"187.19.188.200\" destination-port=\"80\" protocol-id=\"6\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" policy-name=\"argon_policy\" username=\"user1\" session-id-32=\"50000002\" source-zone-name=\"untrust\" destination-zone-name=\"trust\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "BLOCK", + "juniper.srx.file_category": "executable", + "juniper.srx.policy_name": "argon_policy", + "juniper.srx.process": "RT_AAMW", + "juniper.srx.session_id_32": "50000002", + "juniper.srx.tag": "SRX_AAMW_ACTION_LOG", + "juniper.srx.verdict_number": "8", + "juniper.srx.verdict_source": "\u201dcloud/blacklist/whitelist\u201d", + "log.level": "informational", + "log.offset": 0, + "network.iana_number": "6", + "observer.egress.zone": "trust", + "observer.ingress.zone": "untrust", + "observer.name": "pinarello", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "www.mytest.com" + ], + "related.ip": [ + "10.10.10.1", + "187.19.188.200" + ], + "server.ip": "187.19.188.200", + "server.port": 80, + "service.type": "juniper", + "source.ip": "10.10.10.1", + "source.port": 57116, + "source.user.name": "user1", + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "www.mytest.com" + }, + { + "@timestamp": "2016-09-20T15:43:30.330-02:00", + "event.action": "malware_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "timestamp=\"Thu Jun 23 09:55:38 2016\" tenant-id=\"ABC123456\" sample-sha256=\"ABC123\" client-ip=\"192.0.2.0\" verdict-number=\"9\" malware-info=\"Eicar:TestVirus\" username=\"admin\" hostname=\"host.example.com\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.malware_info": "Eicar:TestVirus", + "juniper.srx.process": "RT_AAMW", + "juniper.srx.sample_sha256": "ABC123", + "juniper.srx.tag": "AAMW_MALWARE_EVENT_LOG", + "juniper.srx.tenant_id": "ABC123456", + "juniper.srx.timestamp": "2016-06-23T09:55:38.000Z", + "juniper.srx.verdict_number": "9", + "log.level": "informational", + "log.offset": 529, + "observer.name": "host-example", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "host.example.com" + ], + "related.ip": [ + "192.0.2.0" + ], + "service.type": "juniper", + "source.domain": "host.example.com", + "source.ip": "192.0.2.0", + "source.user.name": "admin", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2016-09-20T15:40:30.050-02:00", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "timestamp=\"Thu Jun 23 09:55:38 2016\" tenant-id=\"ABC123456\" client-ip=\"192.0.2.0\" hostname=\"host.example.com\" status=\"in_progress\" policy-name=\"default\" th=\"7\" state=\"added\" reason=\"malware\" message=\"malware analysis detected host downloaded a malicious_file with score 9, sha256 ABC123\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.message": "malware analysis detected host downloaded a malicious_file with score 9, sha256 ABC123", + "juniper.srx.policy_name": "default", + "juniper.srx.process": "RT_AAMW", + "juniper.srx.reason": "malware", + "juniper.srx.state": "added", + "juniper.srx.status": "in_progress", + "juniper.srx.tag": "AAMW_HOST_INFECTED_EVENT_LOG", + "juniper.srx.tenant_id": "ABC123456", + "juniper.srx.th": "7", + "juniper.srx.timestamp": "2016-06-23T09:55:38.000Z", + "log.level": "error", + "log.offset": 835, + "observer.name": "host-example", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "host.example.com" + ], + "related.ip": [ + "192.0.2.0" + ], + "service.type": "juniper", + "source.domain": "host.example.com", + "source.ip": "192.0.2.0", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2007-02-15T07:17:15.719-02:00", + "client.ip": "1.1.1.1", + "client.port": 60148, + "destination.ip": "10.0.0.1", + "destination.port": 80, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "hostname=\"dummy_host\" file-category=\"executable\" verdict-number=\"10\" malware-info=\"Testfile\" action=\"PERMIT\" list-hit=\"N/A\" file-hash-lookup=\"FALSE\" source-address=\"1.1.1.1\" source-port=\"60148\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" policy-name=\"test-policy\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502156\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" sample-sha256=\"e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494\" file-name=\"dummy_file\" url=\"dummy_url\"", + "event.outcome": "success", + "event.severity": "165", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "PERMIT", + "juniper.srx.application": "HTTP", + "juniper.srx.file_category": "executable", + "juniper.srx.file_hash_lookup": "FALSE", + "juniper.srx.file_name": "dummy_file", + "juniper.srx.malware_info": "Testfile", + "juniper.srx.policy_name": "test-policy", + "juniper.srx.process": "RT_AAMW", + "juniper.srx.sample_sha256": "e038b5168d9209267058112d845341cae83d92b1d1af0a10b66830acb7529494", + "juniper.srx.session_id_32": "502156", + "juniper.srx.tag": "AAMW_ACTION_LOG", + "juniper.srx.url": "dummy_url", + "juniper.srx.verdict_number": "10", + "log.level": "notification", + "log.offset": 1235, + "network.iana_number": "6", + "observer.egress.zone": "Outside", + "observer.ingress.zone": "Inside", + "observer.name": "aamw1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "dummy_host" + ], + "related.ip": [ + "1.1.1.1", + "10.0.0.1" + ], + "server.ip": "10.0.0.1", + "server.port": 80, + "service.type": "juniper", + "source.as.number": 13335, + "source.as.organization.name": "Cloudflare, Inc.", + "source.domain": "dummy_host", + "source.geo.continent_name": "Oceania", + "source.geo.country_iso_code": "AU", + "source.geo.country_name": "Australia", + "source.geo.location.lat": -33.494, + "source.geo.location.lon": 143.2104, + "source.ip": "1.1.1.1", + "source.port": 60148, + "tags": [ + "juniper.srx", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log b/x-pack/filebeat/module/juniper/srx/test/flow.log new file mode 100644 index 00000000000..400bceceeee --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/flow.log @@ -0,0 +1,25 @@ +<14>1 2019-11-14T09:37:51.184+01:00 SRX-GW1 RT_FLOW - RT_FLOW_SESSION_CREATE [junos@2636.1.1.1.2.134 source-address="10.0.0.1" source-port="594" destination-address="10.128.0.1" destination-port="10400" connection-tag="0" service-name="icmp" nat-source-address="10.0.0.1" nat-source-port="594" nat-destination-address="10.128.0.1" nat-destination-port="10400" nat-connection-tag="0" src-nat-rule-type="N/A" src-nat-rule-name="N/A" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="1" policy-name="vpn_trust_permit-all" source-zone-name="vpn" destination-zone-name="trust" session-id-32="6093" username="N/A" roles="N/A" packet-incoming-interface="st0.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A"] +<14>1 2019-11-14T11:12:46.573+01:00 SRX-GW1 RT_FLOW - RT_FLOW_SESSION_DENY [junos@2636.1.1.1.2.134 source-address="10.0.0.26" source-port="37233" destination-address="10.128.0.1" destination-port="161" connection-tag="0" service-name="None" protocol-id="17" icmp-type="0" policy-name="MgmtAccess-trust-cleanup" source-zone-name="trust" destination-zone-name="junos-host" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface=".local..0" encrypted="No" reason="Denied by policy" session-id-32="7087" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A"] +<14>1 2014-05-01T08:26:51.179Z fw01 RT_FLOW - RT_FLOW_SESSION_DENY [junos@2636.1.1.1.2.39 source-address="1.2.3.4" source-port="56639" destination-address="5.6.7.8" destination-port="2003" service-name="None" protocol-id="6" icmp-type="0" policy-name="log-all-else" source-zone-name="campus" destination-zone-name="mngmt" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth6.0" encrypted="No "] +<14>1 2014-05-01T08:28:10.933Z fw01 RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.39 reason="unset" source-address="1.2.3.4" source-port="63456" destination-address="5.6.7.8" destination-port="902" service-name="None" nat-source-address="1.2.3.4" nat-source-port="63456" nat-destination-address="5.6.7.8" nat-destination-port="902" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="17" policy-name="mngmt-to-vcenter" source-zone-name="mngmt" destination-zone-name="intra" session-id-32="15353" packets-from-client="1" bytes-from-client="94" packets-from-server="0" bytes-from-server="0" elapsed-time="60" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth3.5" encrypted="No "] +<14>1 2013-11-04T16:23:09.264Z cixi RT_FLOW - RT_FLOW_SESSION_CREATE [junos@2636.1.1.1.2.35 source-address="50.0.0.100" source-port="24065" destination-address="30.0.0.100" destination-port="768" service-name="icmp" nat-source-address="50.0.0.100" nat-source-port="24065" nat-destination-address="30.0.0.100" nat-destination-port="768" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="1" policy-name="alg-policy" source-zone-name="untrust" destination-zone-name="trust" session-id-32="100000165" username="N/A" roles="N/A" packet-incoming-interface="reth2.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN"] +<14>1 2010-09-30T14:55:04.323+08:00 mrpp-srx550-dut01 RT_FLOW - RT_FLOW_SESSION_CREATE [junos@2626.192.0.2.1.40 source-address="192.0.2.1" source-port="1" destination-address="198.51.100.12" destination-port="46384" service-name="icmp" nat-source-address="192.0.2.1" nat-source-port="1" nat-destination-address="18.51.100.12" nat-destination-port="46384" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="1" policy-name="policy1" source-zone-name="trustZone" destination-zone-name="untrustZone" session-id-32="41" packet-incoming-interface="ge-0/0/1.0"] +<14>1 2010-09-30T14:55:07.188+08:00 mrpp-srx550-dut01 RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2626.192.0.2.1.40 reason="response received" source-address="192.0.2.1" source-port="1" destination-address="198.51.100.12" destination-port="46384" service-name="icmp" nat-source-address="192.0.2.1" nat-source-port="1" nat-destination-address="18.51.100.12" nat-destination-port="46384" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="1" policy-name="policy1" source-zone-name="trustZone" destination-zone-name="untrustZone" session-id-32="41" packets-from-client="1" bytes-from-client="84" packets-from-server="1" bytes-from-server="84" elapsed-time="0" packet-incoming-interface="ge-0/0/1.0"] +<14>1 2019-04-12T14:29:06.576Z cixi RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.129 reason="TCP FIN" source-address="10.3.255.203" source-port="47776" destination-address="8.23.224.110" destination-port="80" connection-tag="0" service-name="junos-http" nat-source-address="10.3.136.49" nat-source-port="19162" nat-destination-address="8.23.224.110" nat-destination-port="80" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="nat1" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="permit_all" source-zone-name="trust" destination-zone-name="untrust" session-id-32="5" packets-from-client="6" bytes-from-client="337" packets-from-server="4" bytes-from-server="535" elapsed-time="1" application="HTTP" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/0.0" encrypted="No" application-category="Web" application-sub-category="N/A" application-risk="4" application-characteristics="Can Leak Information;Supports File Transfer;Prone to Misuse;Known Vulnerabilities;Carrier of Malware;Capable of Tunneling;"] +<14>1 2019-04-13T14:33:06.576Z cixi RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.58 reason="TCP RST" source-address="192.168.2.164" source-port="53232" destination-address="172.16.1.19" destination-port="445" service-name="junos-smb" nat-source-address="192.168.2.164" nat-source-port="53232" nat-destination-address="172.16.1.19" nat-destination-port="445" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="6" policy-name="35" source-zone-name="Trust" destination-zone-name="Trust" session-id-32="206" packets-from-client="13" bytes-from-client="4274" packets-from-server="9" bytes-from-server="1575" elapsed-time="16" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/2.0"] +<14>1 2018-10-07T01:32:20.898Z TestFW2 RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.34 reason="idle Timeout" source-address="100.73.10.92" source-port="52890" destination-address="58.68.126.198" destination-port="53" service-name="junos-dns-udp" nat-source-address="58.78.140.131" nat-source-port="11152" nat-destination-address="58.68.126.198" nat-destination-port="53" src-nat-rule-type="source rule" src-nat-rule-name="NAT_S" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="17" policy-name="NAT" source-zone-name="Gi_nat" destination-zone-name="Internet" session-id-32="220368889" packets-from-client="1" bytes-from-client="72" packets-from-server="1" bytes-from-server="136" elapsed-time="8" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth0.108" encrypted="UNKNOWN"] +<14>1 2018-06-30T02:17:22.753Z fw0001 RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.41 reason="idle Timeout" source-address="192.168.255.2" source-port="62047" destination-address="8.8.8.8" destination-port="53" service-name="junos-dns-udp" nat-source-address="192.168.0.47" nat-source-port="20215" nat-destination-address="8.8.8.8" nat-destination-port="53" src-nat-rule-type="source rule" src-nat-rule-name="rule001" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="17" policy-name="trust-to-untrust-001" source-zone-name="trust" destination-zone-name="untrust" session-id-32="9621" packets-from-client="1" bytes-from-client="67" packets-from-server="1" bytes-from-server="116" elapsed-time="3" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="fe-0/0/1.0" encrypted="UNKNOWN"] +<14>1 2015-09-25T14:19:53.846Z VPNBox-A RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.36 reason="application failure or action" source-address="10.164.110.223" source-port="9057" destination-address="10.104.12.161" destination-port="21" service-name="junos-ftp" nat-source-address="10.9.1.150" nat-source-port="58020" nat-destination-address="10.12.70.1" nat-destination-port="21" src-nat-rule-name="SNAT-Policy5" dst-nat-rule-name="NAT-Policy10" protocol-id="6" policy-name="FW-FTP" source-zone-name="trust" destination-zone-name="untrust" session-id-32="24311" packets-from-client="0" bytes-from-client="0" packets-from-server="0" bytes-from-server="0" elapsed-time="1" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="reth0.0" encrypted="No "] +<14>1 2013-01-19T15:18:17.040 SRX100HM RT_FLOW - APPTRACK_SESSION_CREATE [junos@2636.1.1.1.2.41 source-address="192.168.224.30" source-port="3129" destination-address="207.17.137.56" destination-port="21" service-name="junos-ftp" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="173.167.224.7" nat-source-port="14406" nat-destination-address="207.17.137.56" nat-destination-port="21" src-nat-rule-name="1" dst-nat-rule-name="None" protocol-id="6" policy-name="General-Outbound" source-zone-name="LAN" destination-zone-name="Danger" session-id-32="5058" username="N/A" roles="N/A" encrypted="N/A"] +<14>1 2013-01-19T15:18:17.040 SRX100HM RT_FLOW - APPTRACK_SESSION_VOL_UPDATE [junos@2636.1.1.1.2.41 source-address="192.168.224.30" source-port="3129" destination-address="207.17.137.56" destination-port="21" service-name="junos-ftp" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="173.167.224.7" nat-source-port="14406" nat-destination-address="207.17.137.56" nat-destination-port="21" src-nat-rule-name="1" dst-nat-rule-name="None" protocol-id="6" policy-name="General-Outbound" source-zone-name="LAN" destination-zone-name="Danger" session-id-32="5058" packets-from-client="1" bytes-from-client="48" packets-from-server="0" bytes-from-server="0" elapsed-time="0" username="N/A" roles="N/A" encrypted="N/A"] +<14>1 2013-01-19T15:18:17.040 SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE [junos@2636.1.1.1.2.41 reason="application failure or action" source-address="192.168.224.30" source-port="3129" destination-address="207.17.137.56" destination-port="21" service-name="junos-ftp" application="FTP" nested-application="UNKNOWN" nat-source-address="173.167.224.7" nat-source-port="14406" nat-destination-address="207.17.137.56" nat-destination-port="21" src-nat-rule-name="1" dst-nat-rule-name="None" protocol-id="6" policy-name="General-Outbound" source-zone-name="LAN" destination-zone-name="Danger" session-id-32="5058" packets-from-client="3" bytes-from-client="144" packets-from-server="2" bytes-from-server="104" elapsed-time="1" username="N/A" roles="N/A" encrypted="N/A"] +<14>1 2013-01-19T15:18:18.040 SRX100HM RT_FLOW - APPTRACK_SESSION_VOL_UPDATE [junos@2636.1.1.1.2.129 source-address="4.0.0.1" source-port="33040" destination-address="5.0.0.1" destination-port="80" service-name="junos-http" application="HTTP" nested-application="FACEBOOK-SOCIALRSS" nat-source-address="4.0.0.1" nat-source-port="33040" nat-destination-address="5.0.0.1" nat-destination-port="80" src-nat-rule-name="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="permit-all" source-zone-name="trust" destination-zone-name="untrust" session-id-32="28" packets-from-client="371" bytes-from-client="19592" packets-from-server="584" bytes-from-server="686432" elapsed-time="60" username="user1" roles="DEPT1" encrypted="No" destination-interface-name=”st0.0” apbr-rule-type=”default”] +<14>1 2013-01-19T15:18:19.040 SRX100HM RT_FLOW - APPTRACK_SESSION_ROUTE_UPDATE [junos@2636.1.1.1.2.129 source-address="4.0.0.1" source-port="33040" destination-address="5.0.0.1" destination-port="80" service-name="junos-http" application="HTTP" nested-application="FACEBOOK-SOCIALRSS" nat-source-address="4.0.0.1" nat-source-port="33040" nat-destination-address="5.0.0.1" nat-destination-port="80" src-nat-rule-name="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="permit-all" source-zone-name="trust" destination-zone-name="untrust" session-id-32="28" username="user1" roles="DEPT1" encrypted="No" profile-name=”pf1” rule-name=”facebook1” routing-instance=”instance1” destination-interface-name=”st0.0” apbr-rule-type=”default”] +<14>1 2013-01-19T15:18:20.040 SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE [junos@2636.1.1.1.2.129 reason="TCP CLIENT RST" source-address="4.0.0.1" source-port="48873" destination-address="5.0.0.1" destination-port="80" service-name="junos-http" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="4.0.0.1" nat-source-port="48873" nat-destination-address="5.0.0.1" nat-destination-port="80" src-nat-rule-name="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="permit-all" source-zone-name="trust" destination-zone-name="untrust" session-id-32="32" packets-from-client="5" bytes-from-client="392" packets-from-server="3" bytes-from-server="646" elapsed-time="3" username="user1" roles="DEPT1" encrypted="No" destination-interface-name=”st0.0” apbr-rule-type=”default”] +<14>1 2020-11-04T16:23:09.264Z cixi RT_FLOW - RT_FLOW_SESSION_CREATE_LS [junos@2636.1.1.1.2.35 source-address="50.0.0.100" source-port="24065" destination-address="30.0.0.100" destination-port="768" service-name="icmp" nat-source-address="50.0.0.100" nat-source-port="24065" nat-destination-address="30.0.0.100" nat-destination-port="768" src-nat-rule-name="None" dst-nat-rule-name="None" protocol-id="1" policy-name="alg-policy" source-zone-name="untrust" destination-zone-name="trust" session-id-32="100000165" username="N/A" roles="N/A" packet-incoming-interface="reth2.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN"] +<14>1 2020-11-14T11:12:46.573+01:00 SRX-GW1 RT_FLOW - RT_FLOW_SESSION_DENY_LS [junos@2636.1.1.1.2.134 source-address="10.0.0.26" source-port="37233" destination-address="10.128.0.1" destination-port="161" connection-tag="0" service-name="None" protocol-id="17" icmp-type="0" policy-name="MgmtAccess-trust-cleanup" source-zone-name="trust" destination-zone-name="junos-host" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface=".local..0" encrypted="No" reason="Denied by policy" session-id-32="7087" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A"] +<14>1 2020-01-19T15:18:20.040 SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE_LS [junos@2636.1.1.1.2.129 reason="TCP CLIENT RST" source-address="4.0.0.1" source-port="48873" destination-address="5.0.0.1" destination-port="80" service-name="junos-http" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="4.0.0.1" nat-source-port="48873" nat-destination-address="5.0.0.1" nat-destination-port="80" src-nat-rule-name="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="permit-all" source-zone-name="trust" destination-zone-name="untrust" session-id-32="32" packets-from-client="5" bytes-from-client="392" packets-from-server="3" bytes-from-server="646" elapsed-time="3" username="user1" roles="DEPT1" encrypted="No" destination-interface-name=”st0.0” apbr-rule-type=”default”] +<14>1 2020-07-14T14:17:11.928Z SRX100HM RT_FLOW - APPTRACK_SESSION_VOL_UPDATE [junos@2636.1.1.1.2.129 source-address="10.1.1.100" source-port="58943" destination-address="46.165.154.241" destination-port="80" service-name="junos-http" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="172.19.34.100" nat-source-port="6018" nat-destination-address="46.165.154.241" nat-destination-port="80" src-nat-rule-name="our-nat-rule" dst-nat-rule-name="N/A" protocol-id="6" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="16118" packets-from-client="42" bytes-from-client="2322" packets-from-server="34" bytes-from-server="2132" elapsed-time="60" username="N/A" roles="N/A" encrypted="No" destination-interface-name="ge-0/0/0.0" category="N/A" sub-category="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] +<14>1 2020-07-13T16:43:05.041Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CLOSE [junos@2636.1.1.1.2.129 reason="idle Timeout" source-address="10.1.1.100" source-port="64720" destination-address="91.228.167.172" destination-port="8883" connection-tag="0" service-name="None" nat-source-address="172.19.34.100" nat-source-port="24519" nat-destination-address="91.228.167.172" nat-destination-port="8883" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="6" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="3851" packets-from-client="161" bytes-from-client="9530" packets-from-server="96" bytes-from-server="9670" elapsed-time="23755" application="UNKNOWN" nested-application="UNKNOWN" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" secure-web-proxy-session-type="NA" peer-session-id="0" peer-source-address="0.0.0.0" peer-source-port="0" peer-destination-address="0.0.0.0" peer-destination-port="0" hostname="NA NA" src-vrf-grp="N/A" dst-vrf-grp="N/A"] +<14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - RT_FLOW_SESSION_CREATE [junos@2636.1.1.1.2.129 source-address="10.1.1.100" source-port="49583" destination-address="8.8.8.8" destination-port="53" connection-tag="0" service-name="junos-dns-udp" nat-source-address="172.19.34.100" nat-source-port="30838" nat-destination-address="8.8.8.8" nat-destination-port="53" nat-connection-tag="0" src-nat-rule-type="source rule" src-nat-rule-name="our-nat-rule" dst-nat-rule-type="N/A" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15399" username="N/A" roles="N/A" packet-incoming-interface="ge-0/0/1.0" application="UNKNOWN" nested-application="UNKNOWN" encrypted="UNKNOWN" application-category="N/A" application-sub-category="N/A" application-risk="1" application-characteristics="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] +<14>1 2020-07-13T16:12:05.530Z SRX100HM RT_FLOW - APPTRACK_SESSION_CLOSE [junos@2636.1.1.1.2.129 reason="Closed by junos-alg" source-address="10.1.1.100" source-port="63381" destination-address="8.8.8.8" destination-port="53" service-name="junos-dns-udp" application="UNKNOWN" nested-application="UNKNOWN" nat-source-address="172.19.34.100" nat-source-port="26764" nat-destination-address="8.8.8.8" nat-destination-port="53" src-nat-rule-name="our-nat-rule" dst-nat-rule-name="N/A" protocol-id="17" policy-name="default-permit" source-zone-name="trust" destination-zone-name="untrust" session-id-32="15361" packets-from-client="1" bytes-from-client="66" packets-from-server="1" bytes-from-server="82" elapsed-time="3" username="N/A" roles="N/A" encrypted="No" profile-name="N/A" rule-name="N/A" routing-instance="default" destination-interface-name="ge-0/0/0.0" uplink-incoming-interface-name="N/A" uplink-tx-bytes="0" uplink-rx-bytes="0" category="N/A" sub-category="N/A" apbr-policy-name="N/A" multipath-rule-name="N/A" src-vrf-grp="N/A" dst-vrf-grp="N/A"] diff --git a/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json new file mode 100644 index 00000000000..b597ed2afc5 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/flow.log-expected.json @@ -0,0 +1,2013 @@ +[ + { + "@timestamp": "2019-11-14T06:37:51.184-02:00", + "client.ip": "10.0.0.1", + "client.nat.port": 594, + "client.port": 594, + "destination.ip": "10.128.0.1", + "destination.nat.ip": "10.128.0.1", + "destination.nat.port": 10400, + "destination.port": 10400, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.0.0.1\" source-port=\"594\" destination-address=\"10.128.0.1\" destination-port=\"10400\" connection-tag=\"0\" service-name=\"icmp\" nat-source-address=\"10.0.0.1\" nat-source-port=\"594\" nat-destination-address=\"10.128.0.1\" nat-destination-port=\"10400\" nat-connection-tag=\"0\" src-nat-rule-type=\"N/A\" src-nat-rule-name=\"N/A\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"1\" policy-name=\"vpn_trust_permit-all\" source-zone-name=\"vpn\" destination-zone-name=\"trust\" session-id-32=\"6093\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"st0.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", + "event.outcome": "success", + "event.risk_score": "1", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.nat_connection_tag": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "icmp", + "juniper.srx.session_id_32": "6093", + "juniper.srx.tag": "RT_FLOW_SESSION_CREATE", + "log.level": "informational", + "log.offset": 0, + "network.iana_number": "1", + "observer.egress.zone": "trust", + "observer.ingress.interface.name": "st0.0", + "observer.ingress.zone": "vpn", + "observer.name": "SRX-GW1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.0.0.1", + "10.128.0.1", + "10.0.0.1", + "10.128.0.1" + ], + "rule.name": "vpn_trust_permit-all", + "server.ip": "10.128.0.1", + "server.nat.port": 10400, + "server.port": 10400, + "service.type": "juniper", + "source.ip": "10.0.0.1", + "source.nat.ip": "10.0.0.1", + "source.nat.port": 594, + "source.port": 594, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2019-11-14T08:12:46.573-02:00", + "client.ip": "10.0.0.26", + "client.port": 37233, + "destination.ip": "10.128.0.1", + "destination.port": 161, + "event.action": "flow_deny", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.0.0.26\" source-port=\"37233\" destination-address=\"10.128.0.1\" destination-port=\"161\" connection-tag=\"0\" service-name=\"None\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"MgmtAccess-trust-cleanup\" source-zone-name=\"trust\" destination-zone-name=\"junos-host\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\".local..0\" encrypted=\"No\" reason=\"Denied by policy\" session-id-32=\"7087\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", + "event.outcome": "success", + "event.risk_score": "1", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.encrypted": "No", + "juniper.srx.icmp_type": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "Denied by policy", + "juniper.srx.session_id_32": "7087", + "juniper.srx.tag": "RT_FLOW_SESSION_DENY", + "log.level": "informational", + "log.offset": 850, + "network.iana_number": "17", + "observer.egress.zone": "junos-host", + "observer.ingress.interface.name": ".local..0", + "observer.ingress.zone": "trust", + "observer.name": "SRX-GW1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.0.0.26", + "10.128.0.1" + ], + "rule.name": "MgmtAccess-trust-cleanup", + "server.ip": "10.128.0.1", + "server.port": 161, + "service.type": "juniper", + "source.ip": "10.0.0.26", + "source.port": 37233, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2014-05-01T06:26:51.179-02:00", + "client.ip": "1.2.3.4", + "client.port": 56639, + "destination.as.number": 6805, + "destination.as.organization.name": "Telefonica Germany", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "DE", + "destination.geo.country_name": "Germany", + "destination.geo.location.lat": 51.2993, + "destination.geo.location.lon": 9.491, + "destination.ip": "5.6.7.8", + "destination.port": 2003, + "event.action": "flow_deny", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"1.2.3.4\" source-port=\"56639\" destination-address=\"5.6.7.8\" destination-port=\"2003\" service-name=\"None\" protocol-id=\"6\" icmp-type=\"0\" policy-name=\"log-all-else\" source-zone-name=\"campus\" destination-zone-name=\"mngmt\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth6.0\" encrypted=\"No \"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.encrypted": "No ", + "juniper.srx.icmp_type": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.tag": "RT_FLOW_SESSION_DENY", + "log.level": "informational", + "log.offset": 1513, + "network.iana_number": "6", + "observer.egress.zone": "mngmt", + "observer.ingress.interface.name": "reth6.0", + "observer.ingress.zone": "campus", + "observer.name": "fw01", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "1.2.3.4", + "5.6.7.8" + ], + "rule.name": "log-all-else", + "server.ip": "5.6.7.8", + "server.port": 2003, + "service.type": "juniper", + "source.geo.city_name": "Moscow", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "RU", + "source.geo.country_name": "Russia", + "source.geo.location.lat": 55.7527, + "source.geo.location.lon": 37.6172, + "source.geo.region_iso_code": "RU-MOW", + "source.geo.region_name": "Moscow", + "source.ip": "1.2.3.4", + "source.port": 56639, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2014-05-01T06:28:10.933-02:00", + "client.bytes": 94, + "client.ip": "1.2.3.4", + "client.nat.port": 63456, + "client.packets": 1, + "client.port": 63456, + "destination.as.number": 6805, + "destination.as.organization.name": "Telefonica Germany", + "destination.bytes": 0, + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "DE", + "destination.geo.country_name": "Germany", + "destination.geo.location.lat": 51.2993, + "destination.geo.location.lon": 9.491, + "destination.ip": "5.6.7.8", + "destination.nat.ip": "5.6.7.8", + "destination.nat.port": 902, + "destination.packets": 0, + "destination.port": 902, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 60000000000, + "event.end": "2014-05-01T06:29:10.933-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"unset\" source-address=\"1.2.3.4\" source-port=\"63456\" destination-address=\"5.6.7.8\" destination-port=\"902\" service-name=\"None\" nat-source-address=\"1.2.3.4\" nat-source-port=\"63456\" nat-destination-address=\"5.6.7.8\" nat-destination-port=\"902\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"17\" policy-name=\"mngmt-to-vcenter\" source-zone-name=\"mngmt\" destination-zone-name=\"intra\" session-id-32=\"15353\" packets-from-client=\"1\" bytes-from-client=\"94\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"60\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth3.5\" encrypted=\"No \"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2014-05-01T06:28:10.933-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.encrypted": "No ", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "unset", + "juniper.srx.session_id_32": "15353", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 1966, + "network.bytes": 94, + "network.iana_number": "17", + "network.packets": 1, + "observer.egress.zone": "intra", + "observer.ingress.interface.name": "reth3.5", + "observer.ingress.zone": "mngmt", + "observer.name": "fw01", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "1.2.3.4", + "5.6.7.8", + "1.2.3.4", + "5.6.7.8" + ], + "rule.name": "mngmt-to-vcenter", + "server.bytes": 0, + "server.ip": "5.6.7.8", + "server.nat.port": 902, + "server.packets": 0, + "server.port": 902, + "service.type": "juniper", + "source.bytes": 94, + "source.geo.city_name": "Moscow", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "RU", + "source.geo.country_name": "Russia", + "source.geo.location.lat": 55.7527, + "source.geo.location.lon": 37.6172, + "source.geo.region_iso_code": "RU-MOW", + "source.geo.region_name": "Moscow", + "source.ip": "1.2.3.4", + "source.nat.ip": "1.2.3.4", + "source.nat.port": 63456, + "source.packets": 1, + "source.port": 63456, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-11-04T14:23:09.264-02:00", + "client.ip": "50.0.0.100", + "client.nat.port": 24065, + "client.port": 24065, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "30.0.0.100", + "destination.nat.ip": "30.0.0.100", + "destination.nat.port": 768, + "destination.port": 768, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"50.0.0.100\" source-port=\"24065\" destination-address=\"30.0.0.100\" destination-port=\"768\" service-name=\"icmp\" nat-source-address=\"50.0.0.100\" nat-source-port=\"24065\" nat-destination-address=\"30.0.0.100\" nat-destination-port=\"768\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"alg-policy\" source-zone-name=\"untrust\" destination-zone-name=\"trust\" session-id-32=\"100000165\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth2.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "icmp", + "juniper.srx.session_id_32": "100000165", + "juniper.srx.tag": "RT_FLOW_SESSION_CREATE", + "log.level": "informational", + "log.offset": 2721, + "network.iana_number": "1", + "observer.egress.zone": "trust", + "observer.ingress.interface.name": "reth2.0", + "observer.ingress.zone": "untrust", + "observer.name": "cixi", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "50.0.0.100", + "30.0.0.100", + "50.0.0.100", + "30.0.0.100" + ], + "rule.name": "alg-policy", + "server.ip": "30.0.0.100", + "server.nat.port": 768, + "server.port": 768, + "service.type": "juniper", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "50.0.0.100", + "source.nat.ip": "50.0.0.100", + "source.nat.port": 24065, + "source.port": 24065, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2010-09-30T04:55:04.323-02:00", + "client.ip": "192.0.2.1", + "client.nat.port": 1, + "client.port": 1, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "198.51.100.12", + "destination.nat.ip": "18.51.100.12", + "destination.nat.port": 46384, + "destination.port": 46384, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"192.0.2.1\" source-port=\"1\" destination-address=\"198.51.100.12\" destination-port=\"46384\" service-name=\"icmp\" nat-source-address=\"192.0.2.1\" nat-source-port=\"1\" nat-destination-address=\"18.51.100.12\" nat-destination-port=\"46384\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"policy1\" source-zone-name=\"trustZone\" destination-zone-name=\"untrustZone\" session-id-32=\"41\" packet-incoming-interface=\"ge-0/0/1.0\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "icmp", + "juniper.srx.session_id_32": "41", + "juniper.srx.tag": "RT_FLOW_SESSION_CREATE", + "log.level": "informational", + "log.offset": 3366, + "network.iana_number": "1", + "observer.egress.zone": "untrustZone", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "mrpp-srx550-dut01", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.0.2.1", + "198.51.100.12", + "192.0.2.1", + "18.51.100.12" + ], + "rule.name": "policy1", + "server.ip": "198.51.100.12", + "server.nat.port": 46384, + "server.port": 46384, + "service.type": "juniper", + "source.ip": "192.0.2.1", + "source.nat.ip": "192.0.2.1", + "source.nat.port": 1, + "source.port": 1, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2010-09-30T04:55:07.188-02:00", + "client.bytes": 84, + "client.ip": "192.0.2.1", + "client.nat.port": 1, + "client.packets": 1, + "client.port": 1, + "destination.bytes": 84, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "198.51.100.12", + "destination.nat.ip": "18.51.100.12", + "destination.nat.port": 46384, + "destination.packets": 1, + "destination.port": 46384, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2010-09-30T04:55:07.188-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"response received\" source-address=\"192.0.2.1\" source-port=\"1\" destination-address=\"198.51.100.12\" destination-port=\"46384\" service-name=\"icmp\" nat-source-address=\"192.0.2.1\" nat-source-port=\"1\" nat-destination-address=\"18.51.100.12\" nat-destination-port=\"46384\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"policy1\" source-zone-name=\"trustZone\" destination-zone-name=\"untrustZone\" session-id-32=\"41\" packets-from-client=\"1\" bytes-from-client=\"84\" packets-from-server=\"1\" bytes-from-server=\"84\" elapsed-time=\"0\" packet-incoming-interface=\"ge-0/0/1.0\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2010-09-30T04:55:07.188-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "response received", + "juniper.srx.service_name": "icmp", + "juniper.srx.session_id_32": "41", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 3933, + "network.bytes": 168, + "network.iana_number": "1", + "network.packets": 2, + "observer.egress.zone": "untrustZone", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "mrpp-srx550-dut01", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.0.2.1", + "198.51.100.12", + "192.0.2.1", + "18.51.100.12" + ], + "rule.name": "policy1", + "server.bytes": 84, + "server.ip": "198.51.100.12", + "server.nat.port": 46384, + "server.packets": 1, + "server.port": 46384, + "service.type": "juniper", + "source.bytes": 84, + "source.ip": "192.0.2.1", + "source.nat.ip": "192.0.2.1", + "source.nat.port": 1, + "source.packets": 1, + "source.port": 1, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2019-04-12T12:29:06.576-02:00", + "client.bytes": 337, + "client.ip": "10.3.255.203", + "client.nat.port": 19162, + "client.packets": 6, + "client.port": 47776, + "destination.as.number": 14627, + "destination.as.organization.name": "Vitalwerks Internet Solutions, LLC", + "destination.bytes": 535, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.23.224.110", + "destination.nat.ip": "8.23.224.110", + "destination.nat.port": 80, + "destination.packets": 4, + "destination.port": 80, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 1000000000, + "event.end": "2019-04-12T12:29:07.576-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"TCP FIN\" source-address=\"10.3.255.203\" source-port=\"47776\" destination-address=\"8.23.224.110\" destination-port=\"80\" connection-tag=\"0\" service-name=\"junos-http\" nat-source-address=\"10.3.136.49\" nat-source-port=\"19162\" nat-destination-address=\"8.23.224.110\" nat-destination-port=\"80\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"nat1\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit_all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"5\" packets-from-client=\"6\" bytes-from-client=\"337\" packets-from-server=\"4\" bytes-from-server=\"535\" elapsed-time=\"1\" application=\"HTTP\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/0.0\" encrypted=\"No\" application-category=\"Web\" application-sub-category=\"N/A\" application-risk=\"4\" application-characteristics=\"Can Leak Information;Supports File Transfer;Prone to Misuse;Known Vulnerabilities;Carrier of Malware;Capable of Tunneling;\"", + "event.outcome": "success", + "event.risk_score": "4", + "event.severity": "14", + "event.start": "2019-04-12T12:29:06.576-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.application": "HTTP", + "juniper.srx.application_category": "Web", + "juniper.srx.application_characteristics": "Can Leak Information;Supports File Transfer;Prone to Misuse;Known Vulnerabilities;Carrier of Malware;Capable of Tunneling;", + "juniper.srx.connection_tag": "0", + "juniper.srx.encrypted": "No", + "juniper.srx.nat_connection_tag": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "TCP FIN", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "5", + "juniper.srx.src_nat_rule_name": "nat1", + "juniper.srx.src_nat_rule_type": "source rule", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 4637, + "network.bytes": 872, + "network.iana_number": "6", + "network.packets": 10, + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "ge-0/0/0.0", + "observer.ingress.zone": "trust", + "observer.name": "cixi", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.3.255.203", + "8.23.224.110", + "10.3.136.49", + "8.23.224.110" + ], + "rule.name": "permit_all", + "server.bytes": 535, + "server.ip": "8.23.224.110", + "server.nat.port": 80, + "server.packets": 4, + "server.port": 80, + "service.type": "juniper", + "source.bytes": 337, + "source.ip": "10.3.255.203", + "source.nat.ip": "10.3.136.49", + "source.nat.port": 19162, + "source.packets": 6, + "source.port": 47776, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2019-04-13T12:33:06.576-02:00", + "client.bytes": 4274, + "client.ip": "192.168.2.164", + "client.nat.port": 53232, + "client.packets": 13, + "client.port": 53232, + "destination.bytes": 1575, + "destination.ip": "172.16.1.19", + "destination.nat.ip": "172.16.1.19", + "destination.nat.port": 445, + "destination.packets": 9, + "destination.port": 445, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 16000000000, + "event.end": "2019-04-13T12:33:22.576-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"TCP RST\" source-address=\"192.168.2.164\" source-port=\"53232\" destination-address=\"172.16.1.19\" destination-port=\"445\" service-name=\"junos-smb\" nat-source-address=\"192.168.2.164\" nat-source-port=\"53232\" nat-destination-address=\"172.16.1.19\" nat-destination-port=\"445\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"35\" source-zone-name=\"Trust\" destination-zone-name=\"Trust\" session-id-32=\"206\" packets-from-client=\"13\" bytes-from-client=\"4274\" packets-from-server=\"9\" bytes-from-server=\"1575\" elapsed-time=\"16\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/2.0\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2019-04-13T12:33:06.576-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "TCP RST", + "juniper.srx.service_name": "junos-smb", + "juniper.srx.session_id_32": "206", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 5739, + "network.bytes": 5849, + "network.iana_number": "6", + "network.packets": 22, + "observer.egress.zone": "Trust", + "observer.ingress.interface.name": "ge-0/0/2.0", + "observer.ingress.zone": "Trust", + "observer.name": "cixi", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.2.164", + "172.16.1.19", + "192.168.2.164", + "172.16.1.19" + ], + "rule.name": "35", + "server.bytes": 1575, + "server.ip": "172.16.1.19", + "server.nat.port": 445, + "server.packets": 9, + "server.port": 445, + "service.type": "juniper", + "source.bytes": 4274, + "source.ip": "192.168.2.164", + "source.nat.ip": "192.168.2.164", + "source.nat.port": 53232, + "source.packets": 13, + "source.port": 53232, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-10-06T23:32:20.898-02:00", + "client.bytes": 72, + "client.ip": "100.73.10.92", + "client.nat.port": 11152, + "client.packets": 1, + "client.port": 52890, + "destination.as.number": 10201, + "destination.as.organization.name": "Dishnet Wireless Limited. Broadband Wireless", + "destination.bytes": 136, + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "IN", + "destination.geo.country_name": "India", + "destination.geo.location.lat": 20.0, + "destination.geo.location.lon": 77.0, + "destination.ip": "58.68.126.198", + "destination.nat.ip": "58.68.126.198", + "destination.nat.port": 53, + "destination.packets": 1, + "destination.port": 53, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 8000000000, + "event.end": "2018-10-06T23:32:28.898-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"idle Timeout\" source-address=\"100.73.10.92\" source-port=\"52890\" destination-address=\"58.68.126.198\" destination-port=\"53\" service-name=\"junos-dns-udp\" nat-source-address=\"58.78.140.131\" nat-source-port=\"11152\" nat-destination-address=\"58.68.126.198\" nat-destination-port=\"53\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"NAT_S\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"NAT\" source-zone-name=\"Gi_nat\" destination-zone-name=\"Internet\" session-id-32=\"220368889\" packets-from-client=\"1\" bytes-from-client=\"72\" packets-from-server=\"1\" bytes-from-server=\"136\" elapsed-time=\"8\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.108\" encrypted=\"UNKNOWN\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2018-10-06T23:32:20.898-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "idle Timeout", + "juniper.srx.service_name": "junos-dns-udp", + "juniper.srx.session_id_32": "220368889", + "juniper.srx.src_nat_rule_name": "NAT_S", + "juniper.srx.src_nat_rule_type": "source rule", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 6497, + "network.bytes": 208, + "network.iana_number": "17", + "network.packets": 2, + "observer.egress.zone": "Internet", + "observer.ingress.interface.name": "reth0.108", + "observer.ingress.zone": "Gi_nat", + "observer.name": "TestFW2", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "100.73.10.92", + "58.68.126.198", + "58.78.140.131", + "58.68.126.198" + ], + "rule.name": "NAT", + "server.bytes": 136, + "server.ip": "58.68.126.198", + "server.nat.port": 53, + "server.packets": 1, + "server.port": 53, + "service.type": "juniper", + "source.as.number": 3786, + "source.as.organization.name": "LG DACOM Corporation", + "source.bytes": 72, + "source.geo.city_name": "Seogwipo", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "KR", + "source.geo.country_name": "South Korea", + "source.geo.location.lat": 33.2486, + "source.geo.location.lon": 126.5628, + "source.geo.region_iso_code": "KR-49", + "source.geo.region_name": "Jeju-do", + "source.ip": "100.73.10.92", + "source.nat.ip": "58.78.140.131", + "source.nat.port": 11152, + "source.packets": 1, + "source.port": 52890, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-06-30T00:17:22.753-02:00", + "client.bytes": 67, + "client.ip": "192.168.255.2", + "client.nat.port": 20215, + "client.packets": 1, + "client.port": 62047, + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.bytes": 116, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.nat.ip": "8.8.8.8", + "destination.nat.port": 53, + "destination.packets": 1, + "destination.port": 53, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 3000000000, + "event.end": "2018-06-30T00:17:25.753-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"idle Timeout\" source-address=\"192.168.255.2\" source-port=\"62047\" destination-address=\"8.8.8.8\" destination-port=\"53\" service-name=\"junos-dns-udp\" nat-source-address=\"192.168.0.47\" nat-source-port=\"20215\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"rule001\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"trust-to-untrust-001\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"9621\" packets-from-client=\"1\" bytes-from-client=\"67\" packets-from-server=\"1\" bytes-from-server=\"116\" elapsed-time=\"3\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"fe-0/0/1.0\" encrypted=\"UNKNOWN\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2018-06-30T00:17:22.753-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "idle Timeout", + "juniper.srx.service_name": "junos-dns-udp", + "juniper.srx.session_id_32": "9621", + "juniper.srx.src_nat_rule_name": "rule001", + "juniper.srx.src_nat_rule_type": "source rule", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 7350, + "network.bytes": 183, + "network.iana_number": "17", + "network.packets": 2, + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "fe-0/0/1.0", + "observer.ingress.zone": "trust", + "observer.name": "fw0001", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.255.2", + "8.8.8.8", + "192.168.0.47", + "8.8.8.8" + ], + "rule.name": "trust-to-untrust-001", + "server.bytes": 116, + "server.ip": "8.8.8.8", + "server.nat.port": 53, + "server.packets": 1, + "server.port": 53, + "service.type": "juniper", + "source.bytes": 67, + "source.ip": "192.168.255.2", + "source.nat.ip": "192.168.0.47", + "source.nat.port": 20215, + "source.packets": 1, + "source.port": 62047, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2015-09-25T12:19:53.846-02:00", + "client.bytes": 0, + "client.ip": "10.164.110.223", + "client.nat.port": 58020, + "client.packets": 0, + "client.port": 9057, + "destination.bytes": 0, + "destination.ip": "10.104.12.161", + "destination.nat.ip": "10.12.70.1", + "destination.nat.port": 21, + "destination.packets": 0, + "destination.port": 21, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 1000000000, + "event.end": "2015-09-25T12:19:54.846-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"application failure or action\" source-address=\"10.164.110.223\" source-port=\"9057\" destination-address=\"10.104.12.161\" destination-port=\"21\" service-name=\"junos-ftp\" nat-source-address=\"10.9.1.150\" nat-source-port=\"58020\" nat-destination-address=\"10.12.70.1\" nat-destination-port=\"21\" src-nat-rule-name=\"SNAT-Policy5\" dst-nat-rule-name=\"NAT-Policy10\" protocol-id=\"6\" policy-name=\"FW-FTP\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"24311\" packets-from-client=\"0\" bytes-from-client=\"0\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"1\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth0.0\" encrypted=\"No \"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2015-09-25T12:19:53.846-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.dst_nat_rule_name": "NAT-Policy10", + "juniper.srx.encrypted": "No ", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "application failure or action", + "juniper.srx.service_name": "junos-ftp", + "juniper.srx.session_id_32": "24311", + "juniper.srx.src_nat_rule_name": "SNAT-Policy5", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 8203, + "network.bytes": 0, + "network.iana_number": "6", + "network.packets": 0, + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "reth0.0", + "observer.ingress.zone": "trust", + "observer.name": "VPNBox-A", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.164.110.223", + "10.104.12.161", + "10.9.1.150", + "10.12.70.1" + ], + "rule.name": "FW-FTP", + "server.bytes": 0, + "server.ip": "10.104.12.161", + "server.nat.port": 21, + "server.packets": 0, + "server.port": 21, + "service.type": "juniper", + "source.bytes": 0, + "source.ip": "10.164.110.223", + "source.nat.ip": "10.9.1.150", + "source.nat.port": 58020, + "source.packets": 0, + "source.port": 9057, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:17.040-02:00", + "client.ip": "192.168.224.30", + "client.nat.port": 14406, + "client.port": 3129, + "destination.as.number": 701, + "destination.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "207.17.137.56", + "destination.nat.ip": "207.17.137.56", + "destination.nat.port": 21, + "destination.port": 21, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "junos-ftp", + "juniper.srx.session_id_32": "5058", + "juniper.srx.src_nat_rule_name": "1", + "juniper.srx.tag": "APPTRACK_SESSION_CREATE", + "log.level": "informational", + "log.offset": 9012, + "network.iana_number": "6", + "observer.egress.zone": "Danger", + "observer.ingress.zone": "LAN", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.224.30", + "207.17.137.56", + "173.167.224.7", + "207.17.137.56" + ], + "rule.name": "General-Outbound", + "server.ip": "207.17.137.56", + "server.nat.port": 21, + "server.port": 21, + "service.type": "juniper", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.geo.city_name": "Plymouth", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 42.3695, + "source.geo.location.lon": -83.4769, + "source.geo.region_iso_code": "US-MI", + "source.geo.region_name": "Michigan", + "source.ip": "192.168.224.30", + "source.nat.ip": "173.167.224.7", + "source.nat.port": 14406, + "source.port": 3129, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:17.040-02:00", + "client.bytes": 48, + "client.ip": "192.168.224.30", + "client.nat.port": 14406, + "client.packets": 1, + "client.port": 3129, + "destination.as.number": 701, + "destination.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "destination.bytes": 0, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "207.17.137.56", + "destination.nat.ip": "207.17.137.56", + "destination.nat.port": 21, + "destination.packets": 0, + "destination.port": 21, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2013-01-19T15:18:17.040-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" packets-from-client=\"1\" bytes-from-client=\"48\" packets-from-server=\"0\" bytes-from-server=\"0\" elapsed-time=\"0\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2013-01-19T15:18:17.040-02:00", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "junos-ftp", + "juniper.srx.session_id_32": "5058", + "juniper.srx.src_nat_rule_name": "1", + "juniper.srx.tag": "APPTRACK_SESSION_VOL_UPDATE", + "log.level": "informational", + "log.offset": 9631, + "network.bytes": 48, + "network.iana_number": "6", + "network.packets": 1, + "observer.egress.zone": "Danger", + "observer.ingress.zone": "LAN", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.224.30", + "207.17.137.56", + "173.167.224.7", + "207.17.137.56" + ], + "rule.name": "General-Outbound", + "server.bytes": 0, + "server.ip": "207.17.137.56", + "server.nat.port": 21, + "server.packets": 0, + "server.port": 21, + "service.type": "juniper", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.bytes": 48, + "source.geo.city_name": "Plymouth", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 42.3695, + "source.geo.location.lon": -83.4769, + "source.geo.region_iso_code": "US-MI", + "source.geo.region_name": "Michigan", + "source.ip": "192.168.224.30", + "source.nat.ip": "173.167.224.7", + "source.nat.port": 14406, + "source.packets": 1, + "source.port": 3129, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:17.040-02:00", + "client.bytes": 144, + "client.ip": "192.168.224.30", + "client.nat.port": 14406, + "client.packets": 3, + "client.port": 3129, + "destination.as.number": 701, + "destination.as.organization.name": "MCI Communications Services, Inc. d/b/a Verizon Business", + "destination.bytes": 104, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "207.17.137.56", + "destination.nat.ip": "207.17.137.56", + "destination.nat.port": 21, + "destination.packets": 2, + "destination.port": 21, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 1000000000, + "event.end": "2013-01-19T15:18:18.040-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"application failure or action\" source-address=\"192.168.224.30\" source-port=\"3129\" destination-address=\"207.17.137.56\" destination-port=\"21\" service-name=\"junos-ftp\" application=\"FTP\" nested-application=\"UNKNOWN\" nat-source-address=\"173.167.224.7\" nat-source-port=\"14406\" nat-destination-address=\"207.17.137.56\" nat-destination-port=\"21\" src-nat-rule-name=\"1\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"General-Outbound\" source-zone-name=\"LAN\" destination-zone-name=\"Danger\" session-id-32=\"5058\" packets-from-client=\"3\" bytes-from-client=\"144\" packets-from-server=\"2\" bytes-from-server=\"104\" elapsed-time=\"1\" username=\"N/A\" roles=\"N/A\" encrypted=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2013-01-19T15:18:17.040-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.application": "FTP", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "application failure or action", + "juniper.srx.service_name": "junos-ftp", + "juniper.srx.session_id_32": "5058", + "juniper.srx.src_nat_rule_name": "1", + "juniper.srx.tag": "APPTRACK_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 10364, + "network.bytes": 248, + "network.iana_number": "6", + "network.packets": 5, + "observer.egress.zone": "Danger", + "observer.ingress.zone": "LAN", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.224.30", + "207.17.137.56", + "173.167.224.7", + "207.17.137.56" + ], + "rule.name": "General-Outbound", + "server.bytes": 104, + "server.ip": "207.17.137.56", + "server.nat.port": 21, + "server.packets": 2, + "server.port": 21, + "service.type": "juniper", + "source.as.number": 7922, + "source.as.organization.name": "Comcast Cable Communications, LLC", + "source.bytes": 144, + "source.geo.city_name": "Plymouth", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 42.3695, + "source.geo.location.lon": -83.4769, + "source.geo.region_iso_code": "US-MI", + "source.geo.region_name": "Michigan", + "source.ip": "192.168.224.30", + "source.nat.ip": "173.167.224.7", + "source.nat.port": 14406, + "source.packets": 3, + "source.port": 3129, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:18.040-02:00", + "client.bytes": 19592, + "client.ip": "4.0.0.1", + "client.nat.port": 33040, + "client.packets": 371, + "client.port": 33040, + "destination.as.number": 29256, + "destination.as.organization.name": "Syrian Telecom", + "destination.bytes": 686432, + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "SY", + "destination.geo.country_name": "Syria", + "destination.geo.location.lat": 35.0, + "destination.geo.location.lon": 38.0, + "destination.ip": "5.0.0.1", + "destination.nat.ip": "5.0.0.1", + "destination.nat.port": 80, + "destination.packets": 584, + "destination.port": 80, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 60000000000, + "event.end": "2013-01-19T15:19:18.040-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"4.0.0.1\" source-port=\"33040\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"HTTP\" nested-application=\"FACEBOOK-SOCIALRSS\" nat-source-address=\"4.0.0.1\" nat-source-port=\"33040\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"28\" packets-from-client=\"371\" bytes-from-client=\"19592\" packets-from-server=\"584\" bytes-from-server=\"686432\" elapsed-time=\"60\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2013-01-19T15:18:18.040-02:00", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.apbr_rule_type": "\u201ddefault\u201d", + "juniper.srx.application": "HTTP", + "juniper.srx.encrypted": "No", + "juniper.srx.nested_application": "FACEBOOK-SOCIALRSS", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.roles": "DEPT1", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "28", + "juniper.srx.tag": "APPTRACK_SESSION_VOL_UPDATE", + "log.level": "informational", + "log.offset": 11130, + "network.bytes": 706024, + "network.iana_number": "6", + "network.packets": 955, + "observer.egress.interface.name": "\u201dst0.0\u201d", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "4.0.0.1", + "5.0.0.1", + "4.0.0.1", + "5.0.0.1" + ], + "rule.name": "permit-all", + "server.bytes": 686432, + "server.ip": "5.0.0.1", + "server.nat.port": 80, + "server.packets": 584, + "server.port": 80, + "service.type": "juniper", + "source.as.number": 3356, + "source.as.organization.name": "Level 3 Parent, LLC", + "source.bytes": 19592, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.0.0.1", + "source.nat.ip": "4.0.0.1", + "source.nat.port": 33040, + "source.packets": 371, + "source.port": 33040, + "source.user.name": "user1", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:19.040-02:00", + "client.ip": "4.0.0.1", + "client.nat.port": 33040, + "client.port": 33040, + "destination.as.number": 29256, + "destination.as.organization.name": "Syrian Telecom", + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "SY", + "destination.geo.country_name": "Syria", + "destination.geo.location.lat": 35.0, + "destination.geo.location.lon": 38.0, + "destination.ip": "5.0.0.1", + "destination.nat.ip": "5.0.0.1", + "destination.nat.port": 80, + "destination.port": 80, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"4.0.0.1\" source-port=\"33040\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"HTTP\" nested-application=\"FACEBOOK-SOCIALRSS\" nat-source-address=\"4.0.0.1\" nat-source-port=\"33040\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"28\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" profile-name=\u201dpf1\u201d rule-name=\u201dfacebook1\u201d routing-instance=\u201dinstance1\u201d destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.apbr_rule_type": "\u201ddefault\u201d", + "juniper.srx.application": "HTTP", + "juniper.srx.encrypted": "No", + "juniper.srx.nested_application": "FACEBOOK-SOCIALRSS", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.profile_name": "\u201dpf1\u201d", + "juniper.srx.roles": "DEPT1", + "juniper.srx.routing_instance": "\u201dinstance1\u201d", + "juniper.srx.rule_name": "\u201dfacebook1\u201d", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "28", + "juniper.srx.tag": "APPTRACK_SESSION_ROUTE_UPDATE", + "log.level": "informational", + "log.offset": 11929, + "network.iana_number": "6", + "observer.egress.interface.name": "\u201dst0.0\u201d", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "4.0.0.1", + "5.0.0.1", + "4.0.0.1", + "5.0.0.1" + ], + "rule.name": "permit-all", + "server.ip": "5.0.0.1", + "server.nat.port": 80, + "server.port": 80, + "service.type": "juniper", + "source.as.number": 3356, + "source.as.organization.name": "Level 3 Parent, LLC", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.0.0.1", + "source.nat.ip": "4.0.0.1", + "source.nat.port": 33040, + "source.port": 33040, + "source.user.name": "user1", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2013-01-19T15:18:20.040-02:00", + "client.bytes": 392, + "client.ip": "4.0.0.1", + "client.nat.port": 48873, + "client.packets": 5, + "client.port": 48873, + "destination.as.number": 29256, + "destination.as.organization.name": "Syrian Telecom", + "destination.bytes": 646, + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "SY", + "destination.geo.country_name": "Syria", + "destination.geo.location.lat": 35.0, + "destination.geo.location.lon": 38.0, + "destination.ip": "5.0.0.1", + "destination.nat.ip": "5.0.0.1", + "destination.nat.port": 80, + "destination.packets": 3, + "destination.port": 80, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 3000000000, + "event.end": "2013-01-19T15:18:23.040-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"TCP CLIENT RST\" source-address=\"4.0.0.1\" source-port=\"48873\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"4.0.0.1\" nat-source-port=\"48873\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"32\" packets-from-client=\"5\" bytes-from-client=\"392\" packets-from-server=\"3\" bytes-from-server=\"646\" elapsed-time=\"3\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2013-01-19T15:18:20.040-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.apbr_rule_type": "\u201ddefault\u201d", + "juniper.srx.encrypted": "No", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "TCP CLIENT RST", + "juniper.srx.roles": "DEPT1", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "32", + "juniper.srx.tag": "APPTRACK_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 12689, + "network.bytes": 1038, + "network.iana_number": "6", + "network.packets": 8, + "observer.egress.interface.name": "\u201dst0.0\u201d", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "4.0.0.1", + "5.0.0.1", + "4.0.0.1", + "5.0.0.1" + ], + "rule.name": "permit-all", + "server.bytes": 646, + "server.ip": "5.0.0.1", + "server.nat.port": 80, + "server.packets": 3, + "server.port": 80, + "service.type": "juniper", + "source.as.number": 3356, + "source.as.organization.name": "Level 3 Parent, LLC", + "source.bytes": 392, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.0.0.1", + "source.nat.ip": "4.0.0.1", + "source.nat.port": 48873, + "source.packets": 5, + "source.port": 48873, + "source.user.name": "user1", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-11-04T14:23:09.264-02:00", + "client.ip": "50.0.0.100", + "client.nat.port": 24065, + "client.port": 24065, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "30.0.0.100", + "destination.nat.ip": "30.0.0.100", + "destination.nat.port": 768, + "destination.port": 768, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"50.0.0.100\" source-port=\"24065\" destination-address=\"30.0.0.100\" destination-port=\"768\" service-name=\"icmp\" nat-source-address=\"50.0.0.100\" nat-source-port=\"24065\" nat-destination-address=\"30.0.0.100\" nat-destination-port=\"768\" src-nat-rule-name=\"None\" dst-nat-rule-name=\"None\" protocol-id=\"1\" policy-name=\"alg-policy\" source-zone-name=\"untrust\" destination-zone-name=\"trust\" session-id-32=\"100000165\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"reth2.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "icmp", + "juniper.srx.session_id_32": "100000165", + "juniper.srx.tag": "RT_FLOW_SESSION_CREATE_LS", + "log.level": "informational", + "log.offset": 13489, + "network.iana_number": "1", + "observer.egress.zone": "trust", + "observer.ingress.interface.name": "reth2.0", + "observer.ingress.zone": "untrust", + "observer.name": "cixi", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "50.0.0.100", + "30.0.0.100", + "50.0.0.100", + "30.0.0.100" + ], + "rule.name": "alg-policy", + "server.ip": "30.0.0.100", + "server.nat.port": 768, + "server.port": 768, + "service.type": "juniper", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "50.0.0.100", + "source.nat.ip": "50.0.0.100", + "source.nat.port": 24065, + "source.port": 24065, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-11-14T08:12:46.573-02:00", + "client.ip": "10.0.0.26", + "client.port": 37233, + "destination.ip": "10.128.0.1", + "destination.port": 161, + "event.action": "flow_deny", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.0.0.26\" source-port=\"37233\" destination-address=\"10.128.0.1\" destination-port=\"161\" connection-tag=\"0\" service-name=\"None\" protocol-id=\"17\" icmp-type=\"0\" policy-name=\"MgmtAccess-trust-cleanup\" source-zone-name=\"trust\" destination-zone-name=\"junos-host\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\".local..0\" encrypted=\"No\" reason=\"Denied by policy\" session-id-32=\"7087\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\"", + "event.outcome": "success", + "event.risk_score": "1", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.encrypted": "No", + "juniper.srx.icmp_type": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "Denied by policy", + "juniper.srx.session_id_32": "7087", + "juniper.srx.tag": "RT_FLOW_SESSION_DENY_LS", + "log.level": "informational", + "log.offset": 14137, + "network.iana_number": "17", + "observer.egress.zone": "junos-host", + "observer.ingress.interface.name": ".local..0", + "observer.ingress.zone": "trust", + "observer.name": "SRX-GW1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.0.0.26", + "10.128.0.1" + ], + "rule.name": "MgmtAccess-trust-cleanup", + "server.ip": "10.128.0.1", + "server.port": 161, + "service.type": "juniper", + "source.ip": "10.0.0.26", + "source.port": 37233, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-01-19T15:18:20.040-02:00", + "client.bytes": 392, + "client.ip": "4.0.0.1", + "client.nat.port": 48873, + "client.packets": 5, + "client.port": 48873, + "destination.as.number": 29256, + "destination.as.organization.name": "Syrian Telecom", + "destination.bytes": 646, + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "SY", + "destination.geo.country_name": "Syria", + "destination.geo.location.lat": 35.0, + "destination.geo.location.lon": 38.0, + "destination.ip": "5.0.0.1", + "destination.nat.ip": "5.0.0.1", + "destination.nat.port": 80, + "destination.packets": 3, + "destination.port": 80, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 3000000000, + "event.end": "2020-01-19T15:18:23.040-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"TCP CLIENT RST\" source-address=\"4.0.0.1\" source-port=\"48873\" destination-address=\"5.0.0.1\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"4.0.0.1\" nat-source-port=\"48873\" nat-destination-address=\"5.0.0.1\" nat-destination-port=\"80\" src-nat-rule-name=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"permit-all\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"32\" packets-from-client=\"5\" bytes-from-client=\"392\" packets-from-server=\"3\" bytes-from-server=\"646\" elapsed-time=\"3\" username=\"user1\" roles=\"DEPT1\" encrypted=\"No\" destination-interface-name=\u201dst0.0\u201d apbr-rule-type=\u201ddefault\u201d", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2020-01-19T15:18:20.040-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.apbr_rule_type": "\u201ddefault\u201d", + "juniper.srx.encrypted": "No", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "TCP CLIENT RST", + "juniper.srx.roles": "DEPT1", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "32", + "juniper.srx.tag": "APPTRACK_SESSION_CLOSE_LS", + "log.level": "informational", + "log.offset": 14803, + "network.bytes": 1038, + "network.iana_number": "6", + "network.packets": 8, + "observer.egress.interface.name": "\u201dst0.0\u201d", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "4.0.0.1", + "5.0.0.1", + "4.0.0.1", + "5.0.0.1" + ], + "rule.name": "permit-all", + "server.bytes": 646, + "server.ip": "5.0.0.1", + "server.nat.port": 80, + "server.packets": 3, + "server.port": 80, + "service.type": "juniper", + "source.as.number": 3356, + "source.as.organization.name": "Level 3 Parent, LLC", + "source.bytes": 392, + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "4.0.0.1", + "source.nat.ip": "4.0.0.1", + "source.nat.port": 48873, + "source.packets": 5, + "source.port": 48873, + "source.user.name": "user1", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-14T12:17:11.928-02:00", + "client.bytes": 2322, + "client.ip": "10.1.1.100", + "client.nat.port": 6018, + "client.packets": 42, + "client.port": 58943, + "destination.as.number": 42652, + "destination.as.organization.name": "inexio Informationstechnologie und Telekommunikation Gmbh", + "destination.bytes": 2132, + "destination.geo.city_name": "Philippsburg", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "DE", + "destination.geo.country_name": "Germany", + "destination.geo.location.lat": 49.2317, + "destination.geo.location.lon": 8.4607, + "destination.geo.region_iso_code": "DE-BW", + "destination.geo.region_name": "Baden-W\u00fcrttemberg", + "destination.ip": "46.165.154.241", + "destination.nat.ip": "46.165.154.241", + "destination.nat.port": 80, + "destination.packets": 34, + "destination.port": 80, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 60000000000, + "event.end": "2020-07-14T12:18:11.928-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.1.1.100\" source-port=\"58943\" destination-address=\"46.165.154.241\" destination-port=\"80\" service-name=\"junos-http\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"172.19.34.100\" nat-source-port=\"6018\" nat-destination-address=\"46.165.154.241\" nat-destination-port=\"80\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"16118\" packets-from-client=\"42\" bytes-from-client=\"2322\" packets-from-server=\"34\" bytes-from-server=\"2132\" elapsed-time=\"60\" username=\"N/A\" roles=\"N/A\" encrypted=\"No\" destination-interface-name=\"ge-0/0/0.0\" category=\"N/A\" sub-category=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2020-07-14T12:17:11.928-02:00", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.encrypted": "No", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "junos-http", + "juniper.srx.session_id_32": "16118", + "juniper.srx.src_nat_rule_name": "our-nat-rule", + "juniper.srx.tag": "APPTRACK_SESSION_VOL_UPDATE", + "log.level": "informational", + "log.offset": 15606, + "network.bytes": 4454, + "network.iana_number": "6", + "network.packets": 76, + "observer.egress.interface.name": "ge-0/0/0.0", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "46.165.154.241", + "172.19.34.100", + "46.165.154.241" + ], + "rule.name": "default-permit", + "server.bytes": 2132, + "server.ip": "46.165.154.241", + "server.nat.port": 80, + "server.packets": 34, + "server.port": 80, + "service.type": "juniper", + "source.bytes": 2322, + "source.ip": "10.1.1.100", + "source.nat.ip": "172.19.34.100", + "source.nat.port": 6018, + "source.packets": 42, + "source.port": 58943, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-13T14:43:05.041-02:00", + "client.bytes": 9530, + "client.ip": "10.1.1.100", + "client.nat.port": 24519, + "client.packets": 161, + "client.port": 64720, + "destination.as.number": 50881, + "destination.as.organization.name": "ESET, spol. s r.o.", + "destination.bytes": 9670, + "destination.geo.city_name": "Bratislava", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "SK", + "destination.geo.country_name": "Slovakia", + "destination.geo.location.lat": 48.15, + "destination.geo.location.lon": 17.1078, + "destination.geo.region_iso_code": "SK-BL", + "destination.geo.region_name": "Bratislava", + "destination.ip": "91.228.167.172", + "destination.nat.ip": "91.228.167.172", + "destination.nat.port": 8883, + "destination.packets": 96, + "destination.port": 8883, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 23755000000000, + "event.end": "2020-07-13T21:19:00.041-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"idle Timeout\" source-address=\"10.1.1.100\" source-port=\"64720\" destination-address=\"91.228.167.172\" destination-port=\"8883\" connection-tag=\"0\" service-name=\"None\" nat-source-address=\"172.19.34.100\" nat-source-port=\"24519\" nat-destination-address=\"91.228.167.172\" nat-destination-port=\"8883\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"6\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"3851\" packets-from-client=\"161\" bytes-from-client=\"9530\" packets-from-server=\"96\" bytes-from-server=\"9670\" elapsed-time=\"23755\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/1.0\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\" secure-web-proxy-session-type=\"NA\" peer-session-id=\"0\" peer-source-address=\"0.0.0.0\" peer-source-port=\"0\" peer-destination-address=\"0.0.0.0\" peer-destination-port=\"0\" hostname=\"NA NA\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", + "event.outcome": "success", + "event.risk_score": "1", + "event.severity": "14", + "event.start": "2020-07-13T14:43:05.041-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.hostname": "NA NA", + "juniper.srx.nat_connection_tag": "0", + "juniper.srx.peer_destination_address": "0.0.0.0", + "juniper.srx.peer_destination_port": "0", + "juniper.srx.peer_session_id": "0", + "juniper.srx.peer_source_address": "0.0.0.0", + "juniper.srx.peer_source_port": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "idle Timeout", + "juniper.srx.secure_web_proxy_session_type": "NA", + "juniper.srx.session_id_32": "3851", + "juniper.srx.src_nat_rule_name": "our-nat-rule", + "juniper.srx.src_nat_rule_type": "source rule", + "juniper.srx.tag": "RT_FLOW_SESSION_CLOSE", + "log.level": "informational", + "log.offset": 16469, + "network.bytes": 19200, + "network.iana_number": "6", + "network.packets": 257, + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "91.228.167.172", + "172.19.34.100", + "91.228.167.172" + ], + "rule.name": "default-permit", + "server.bytes": 9670, + "server.ip": "91.228.167.172", + "server.nat.port": 8883, + "server.packets": 96, + "server.port": 8883, + "service.type": "juniper", + "source.bytes": 9530, + "source.ip": "10.1.1.100", + "source.nat.ip": "172.19.34.100", + "source.nat.port": 24519, + "source.packets": 161, + "source.port": 64720, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-13T14:12:05.530-02:00", + "client.ip": "10.1.1.100", + "client.nat.port": 30838, + "client.port": 49583, + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.nat.ip": "8.8.8.8", + "destination.nat.port": 53, + "destination.port": 53, + "event.action": "flow_started", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.1.1.100\" source-port=\"49583\" destination-address=\"8.8.8.8\" destination-port=\"53\" connection-tag=\"0\" service-name=\"junos-dns-udp\" nat-source-address=\"172.19.34.100\" nat-source-port=\"30838\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" nat-connection-tag=\"0\" src-nat-rule-type=\"source rule\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-type=\"N/A\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"15399\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"ge-0/0/1.0\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" encrypted=\"UNKNOWN\" application-category=\"N/A\" application-sub-category=\"N/A\" application-risk=\"1\" application-characteristics=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", + "event.outcome": "success", + "event.risk_score": "1", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "start", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.connection_tag": "0", + "juniper.srx.nat_connection_tag": "0", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.service_name": "junos-dns-udp", + "juniper.srx.session_id_32": "15399", + "juniper.srx.src_nat_rule_name": "our-nat-rule", + "juniper.srx.src_nat_rule_type": "source rule", + "juniper.srx.tag": "RT_FLOW_SESSION_CREATE", + "log.level": "informational", + "log.offset": 17715, + "network.iana_number": "17", + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "8.8.8.8", + "172.19.34.100", + "8.8.8.8" + ], + "rule.name": "default-permit", + "server.ip": "8.8.8.8", + "server.nat.port": 53, + "server.port": 53, + "service.type": "juniper", + "source.ip": "10.1.1.100", + "source.nat.ip": "172.19.34.100", + "source.nat.port": 30838, + "source.port": 49583, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-13T14:12:05.530-02:00", + "client.bytes": 66, + "client.ip": "10.1.1.100", + "client.nat.port": 26764, + "client.packets": 1, + "client.port": 63381, + "destination.as.number": 15169, + "destination.as.organization.name": "Google LLC", + "destination.bytes": 82, + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "8.8.8.8", + "destination.nat.ip": "8.8.8.8", + "destination.nat.port": 53, + "destination.packets": 1, + "destination.port": 53, + "event.action": "flow_close", + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.duration": 3000000000, + "event.end": "2020-07-13T14:12:08.530-02:00", + "event.kind": "event", + "event.module": "juniper", + "event.original": "reason=\"Closed by junos-alg\" source-address=\"10.1.1.100\" source-port=\"63381\" destination-address=\"8.8.8.8\" destination-port=\"53\" service-name=\"junos-dns-udp\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" nat-source-address=\"172.19.34.100\" nat-source-port=\"26764\" nat-destination-address=\"8.8.8.8\" nat-destination-port=\"53\" src-nat-rule-name=\"our-nat-rule\" dst-nat-rule-name=\"N/A\" protocol-id=\"17\" policy-name=\"default-permit\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"15361\" packets-from-client=\"1\" bytes-from-client=\"66\" packets-from-server=\"1\" bytes-from-server=\"82\" elapsed-time=\"3\" username=\"N/A\" roles=\"N/A\" encrypted=\"No\" profile-name=\"N/A\" rule-name=\"N/A\" routing-instance=\"default\" destination-interface-name=\"ge-0/0/0.0\" uplink-incoming-interface-name=\"N/A\" uplink-tx-bytes=\"0\" uplink-rx-bytes=\"0\" category=\"N/A\" sub-category=\"N/A\" apbr-policy-name=\"N/A\" multipath-rule-name=\"N/A\" src-vrf-grp=\"N/A\" dst-vrf-grp=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.start": "2020-07-13T14:12:05.530-02:00", + "event.timezone": "-02:00", + "event.type": [ + "end", + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.encrypted": "No", + "juniper.srx.process": "RT_FLOW", + "juniper.srx.reason": "Closed by junos-alg", + "juniper.srx.routing_instance": "default", + "juniper.srx.service_name": "junos-dns-udp", + "juniper.srx.session_id_32": "15361", + "juniper.srx.src_nat_rule_name": "our-nat-rule", + "juniper.srx.tag": "APPTRACK_SESSION_CLOSE", + "juniper.srx.uplink_rx_bytes": "0", + "juniper.srx.uplink_tx_bytes": "0", + "log.level": "informational", + "log.offset": 18627, + "network.bytes": 148, + "network.iana_number": "17", + "network.packets": 2, + "observer.egress.interface.name": "ge-0/0/0.0", + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX100HM", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "8.8.8.8", + "172.19.34.100", + "8.8.8.8" + ], + "rule.name": "default-permit", + "server.bytes": 82, + "server.ip": "8.8.8.8", + "server.nat.port": 53, + "server.packets": 1, + "server.port": 53, + "service.type": "juniper", + "source.bytes": 66, + "source.ip": "10.1.1.100", + "source.nat.ip": "172.19.34.100", + "source.nat.port": 26764, + "source.packets": 1, + "source.port": 63381, + "tags": [ + "juniper.srx", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/juniper/srx/test/idp.log b/x-pack/filebeat/module/juniper/srx/test/idp.log new file mode 100644 index 00000000000..c05d9732fb5 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/idp.log @@ -0,0 +1,7 @@ +<165>1 2020-03-02T23:13:03.193Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.28 epoch-time="1583190783" message-type="SIG" source-address="10.11.11.1" source-port="12345" destination-address="187.188.188.10" destination-port="123" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="3" rulebase-name="IPS" policy-name="Recommended" export-id="20175" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="HTTP:MISC:GENERIC-DIR-TRAVERSAL" nat-source-address="0.0.0.0" nat-source-port="13312" nat-destination-address="3.3.10.11" nat-destination-port="9757" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="UNTRUST" source-interface-name="reth1.24" destination-zone-name="DMZ" destination-interface-name="reth2.21" packet-log-id="0" alert="no" username="unknown-user" roles="N/A" index="cnm" type="idp" message="-"] +<165>1 2020-03-02T23:13:03.197Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.28 epoch-time="1583190783" message-type="SIG" source-address="10.11.11.1" source-port="12345" destination-address="187.188.188.10" destination-port="123" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="3" rulebase-name="IPS" policy-name="Recommended" export-id="20175" repeat-count="0" action="DROP" threat-severity="CRITICAL" attack-name="TCP:C2S:AMBIG:C2S-SYN-DATA" nat-source-address="0.0.0.0" nat-source-port="13312" nat-destination-address="3.3.10.11" nat-destination-port="9757" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="UNTRUST" source-interface-name="reth1.24" destination-zone-name="DMZ" destination-interface-name="reth2.21" packet-log-id="0" alert="no" username="unknown-user" roles="N/A" index="cnm" type="idp" message="-"] +<165>1 2007-02-15T09:17:15.719Z idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.111.1" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.19.13.11" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.11" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.1" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] +<165>1 2017-10-13T08:55:55.792+11:00 idp1 RT_IDP - IDP_ATTACK_LOG_EVENT [junos@2636.1.1.1.2.135 epoch-time="1507845354" message-type="SIG" source-address="183.78.180.27" source-port="45610" destination-address="118.127.30.11" destination-port="80" protocol-name="TCP" service-name="SERVICE_IDP" application-name="HTTP" rule-name="9" rulebase-name="IPS" policy-name="Recommended" export-id="15229" repeat-count="0" action="DROP" threat-severity="HIGH" attack-name="TROJAN:ZMEU-BOT-SCAN" nat-source-address="0.0.0.0" nat-source-port="0" nat-destination-address="172.16.1.10" nat-destination-port="0" elapsed-time="0" inbound-bytes="0" outbound-bytes="0" inbound-packets="0" outbound-packets="0" source-zone-name="sec-zone-name-internet" source-interface-name="reth0.11" destination-zone-name="dst-sec-zone1-outside" destination-interface-name="reth1.1" packet-log-id="0" alert="no" username="N/A" roles="N/A" message="-"] +<165>1 2011-10-23T02:06:26.544 SRX34001 RT_IDP - IDP_APPDDOS_APP_STATE_EVENT [junos@2636.1.1.1.2.35 epoch-time="1319367986" ddos-application-name="Webserver" destination-zone-name="untrust" destination-interface-name="reth0.0" destination-address="172.27.14.203" destination-port="80" protocol-name="TCP" service-name="HTTP" rule-name="1" rulebase-name="DDOS" policy-name="A DoS-Webserver" repeat-count="0" message="Connection rate exceeded limit 60" context-value="N/A"] +<165>1 2011-10-23T16:28:31.696 SRX34001 RT_IDP - IDP_APPDDOS_APP_ATTACK_EVENT [junos@2636.1.1.1.2.35 epoch-time="1319419711" ddos-application-name="Webserver" source-zone-name="trust" source-interface-name="reth1.O" source-address="192.168.14.214" source-port="50825" destination-zone-name="untrust" destination-interface-name="reth0.0" destination-address="172.27.14.203" destination-port="80" protocol-name="TCP" service-name="HTTP" rule-name="1" ruleebase-name="DDOS" policy-name="AppDoS-Webserver" repeat-count="0" action="NONE" threat-severity="INFO" connection-hit-rate="30" context-name="http-get-url" context-hit-rate="123" context-value-hit-rate="0" time-scope="PEER" time-count="3" time-period="60" context-value="N/A"] +<165>1 2012-10-23T17:28:31.696 SRX34001 RT_IDP - IDP_APPDDOS_APP_ATTACK_EVENT_LS [junos@2636.1.1.1.2.35 epoch-time="1419419711" ddos-application-name="Webserver" source-zone-name="trust" source-interface-name="reth3.0" source-address="193.168.14.214" source-port="50825" destination-zone-name="untrust" destination-interface-name="reth0.1" destination-address="172.30.20.201" destination-port="80" protocol-name="TCP" service-name="HTTP" rule-name="1" ruleebase-name="DDOS02" policy-name="AppDoS-Webserver" repeat-count="0" action="NONE" threat-severity="INFO" connection-hit-rate="30" context-name="http-get-url" context-hit-rate="123" context-value-hit-rate="0" time-scope="PEER" time-count="3" time-period="60" context-value="N/A"] diff --git a/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json new file mode 100644 index 00000000000..7704c88fac0 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/idp.log-expected.json @@ -0,0 +1,537 @@ +[ + { + "@timestamp": "2020-03-02T21:13:03.193-02:00", + "client.bytes": 0, + "client.ip": "10.11.11.1", + "client.nat.port": 13312, + "client.packets": 0, + "client.port": 12345, + "destination.bytes": 0, + "destination.ip": "187.188.188.10", + "destination.nat.ip": "3.3.10.11", + "destination.nat.port": 9757, + "destination.packets": 0, + "destination.port": 123, + "event.action": "security_threat", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2020-03-02T21:13:03.193-02:00", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1583190783\" message-type=\"SIG\" source-address=\"10.11.11.1\" source-port=\"12345\" destination-address=\"187.188.188.10\" destination-port=\"123\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"3\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"20175\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"HTTP:MISC:GENERIC-DIR-TRAVERSAL\" nat-source-address=\"0.0.0.0\" nat-source-port=\"13312\" nat-destination-address=\"3.3.10.11\" nat-destination-port=\"9757\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"UNTRUST\" source-interface-name=\"reth1.24\" destination-zone-name=\"DMZ\" destination-interface-name=\"reth2.21\" packet-log-id=\"0\" alert=\"no\" username=\"unknown-user\" roles=\"N/A\" index=\"cnm\" type=\"idp\" message=\"-\"", + "event.outcome": "success", + "event.severity": "165", + "event.start": "2020-03-02T21:13:03.193-02:00", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "DROP", + "juniper.srx.alert": "no", + "juniper.srx.application_name": "HTTP", + "juniper.srx.attack_name": "HTTP:MISC:GENERIC-DIR-TRAVERSAL", + "juniper.srx.epoch_time": "1583190783", + "juniper.srx.export_id": "20175", + "juniper.srx.index": "cnm", + "juniper.srx.message_type": "SIG", + "juniper.srx.packet_log_id": "0", + "juniper.srx.policy_name": "Recommended", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.service_name": "SERVICE_IDP", + "juniper.srx.tag": "IDP_ATTACK_LOG_EVENT", + "juniper.srx.threat_severity": "HIGH", + "juniper.srx.type": "idp", + "log.level": "notification", + "log.offset": 0, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth2.21", + "observer.egress.zone": "DMZ", + "observer.ingress.interface.name": "reth1.24", + "observer.ingress.zone": "UNTRUST", + "observer.name": "idp1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.11.11.1", + "187.188.188.10", + "0.0.0.0", + "3.3.10.11" + ], + "rule.id": "3", + "rule.name": "IPS", + "server.bytes": 0, + "server.ip": "187.188.188.10", + "server.nat.port": 9757, + "server.packets": 0, + "server.port": 123, + "service.type": "juniper", + "source.bytes": 0, + "source.ip": "10.11.11.1", + "source.nat.ip": "0.0.0.0", + "source.nat.port": 13312, + "source.packets": 0, + "source.port": 12345, + "source.user.name": "unknown-user", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-03-02T21:13:03.197-02:00", + "client.bytes": 0, + "client.ip": "10.11.11.1", + "client.nat.port": 13312, + "client.packets": 0, + "client.port": 12345, + "destination.bytes": 0, + "destination.ip": "187.188.188.10", + "destination.nat.ip": "3.3.10.11", + "destination.nat.port": 9757, + "destination.packets": 0, + "destination.port": 123, + "event.action": "security_threat", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2020-03-02T21:13:03.197-02:00", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1583190783\" message-type=\"SIG\" source-address=\"10.11.11.1\" source-port=\"12345\" destination-address=\"187.188.188.10\" destination-port=\"123\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"3\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"20175\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"CRITICAL\" attack-name=\"TCP:C2S:AMBIG:C2S-SYN-DATA\" nat-source-address=\"0.0.0.0\" nat-source-port=\"13312\" nat-destination-address=\"3.3.10.11\" nat-destination-port=\"9757\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"UNTRUST\" source-interface-name=\"reth1.24\" destination-zone-name=\"DMZ\" destination-interface-name=\"reth2.21\" packet-log-id=\"0\" alert=\"no\" username=\"unknown-user\" roles=\"N/A\" index=\"cnm\" type=\"idp\" message=\"-\"", + "event.outcome": "success", + "event.severity": "165", + "event.start": "2020-03-02T21:13:03.197-02:00", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "DROP", + "juniper.srx.alert": "no", + "juniper.srx.application_name": "HTTP", + "juniper.srx.attack_name": "TCP:C2S:AMBIG:C2S-SYN-DATA", + "juniper.srx.epoch_time": "1583190783", + "juniper.srx.export_id": "20175", + "juniper.srx.index": "cnm", + "juniper.srx.message_type": "SIG", + "juniper.srx.packet_log_id": "0", + "juniper.srx.policy_name": "Recommended", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.service_name": "SERVICE_IDP", + "juniper.srx.tag": "IDP_ATTACK_LOG_EVENT", + "juniper.srx.threat_severity": "CRITICAL", + "juniper.srx.type": "idp", + "log.level": "notification", + "log.offset": 929, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth2.21", + "observer.egress.zone": "DMZ", + "observer.ingress.interface.name": "reth1.24", + "observer.ingress.zone": "UNTRUST", + "observer.name": "idp1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.11.11.1", + "187.188.188.10", + "0.0.0.0", + "3.3.10.11" + ], + "rule.id": "3", + "rule.name": "IPS", + "server.bytes": 0, + "server.ip": "187.188.188.10", + "server.nat.port": 9757, + "server.packets": 0, + "server.port": 123, + "service.type": "juniper", + "source.bytes": 0, + "source.ip": "10.11.11.1", + "source.nat.ip": "0.0.0.0", + "source.nat.port": 13312, + "source.packets": 0, + "source.port": 12345, + "source.user.name": "unknown-user", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2007-02-15T07:17:15.719-02:00", + "client.bytes": 0, + "client.ip": "183.78.180.27", + "client.nat.port": 0, + "client.packets": 0, + "client.port": 45610, + "destination.bytes": 0, + "destination.ip": "118.127.111.1", + "destination.nat.ip": "172.19.13.11", + "destination.nat.port": 0, + "destination.packets": 0, + "destination.port": 80, + "event.action": "security_threat", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2007-02-15T07:17:15.719-02:00", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.111.1\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.19.13.11\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.11\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.1\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"", + "event.outcome": "success", + "event.severity": "165", + "event.start": "2007-02-15T07:17:15.719-02:00", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "DROP", + "juniper.srx.alert": "no", + "juniper.srx.application_name": "HTTP", + "juniper.srx.attack_name": "TROJAN:ZMEU-BOT-SCAN", + "juniper.srx.epoch_time": "1507845354", + "juniper.srx.export_id": "15229", + "juniper.srx.message_type": "SIG", + "juniper.srx.packet_log_id": "0", + "juniper.srx.policy_name": "Recommended", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.service_name": "SERVICE_IDP", + "juniper.srx.tag": "IDP_ATTACK_LOG_EVENT", + "juniper.srx.threat_severity": "HIGH", + "log.level": "notification", + "log.offset": 1857, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth1.1", + "observer.egress.zone": "dst-sec-zone1-outside", + "observer.ingress.interface.name": "reth0.11", + "observer.ingress.zone": "sec-zone-name-internet", + "observer.name": "idp1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "183.78.180.27", + "118.127.111.1", + "0.0.0.0", + "172.19.13.11" + ], + "rule.id": "9", + "rule.name": "IPS", + "server.bytes": 0, + "server.ip": "118.127.111.1", + "server.nat.port": 0, + "server.packets": 0, + "server.port": 80, + "service.type": "juniper", + "source.bytes": 0, + "source.ip": "183.78.180.27", + "source.nat.ip": "0.0.0.0", + "source.nat.port": 0, + "source.packets": 0, + "source.port": 45610, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2017-10-12T19:55:55.792-02:00", + "client.bytes": 0, + "client.ip": "183.78.180.27", + "client.nat.port": 0, + "client.packets": 0, + "client.port": 45610, + "destination.bytes": 0, + "destination.ip": "118.127.30.11", + "destination.nat.ip": "172.16.1.10", + "destination.nat.port": 0, + "destination.packets": 0, + "destination.port": 80, + "event.action": "security_threat", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.duration": 0, + "event.end": "2017-10-12T19:55:55.792-02:00", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1507845354\" message-type=\"SIG\" source-address=\"183.78.180.27\" source-port=\"45610\" destination-address=\"118.127.30.11\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"SERVICE_IDP\" application-name=\"HTTP\" rule-name=\"9\" rulebase-name=\"IPS\" policy-name=\"Recommended\" export-id=\"15229\" repeat-count=\"0\" action=\"DROP\" threat-severity=\"HIGH\" attack-name=\"TROJAN:ZMEU-BOT-SCAN\" nat-source-address=\"0.0.0.0\" nat-source-port=\"0\" nat-destination-address=\"172.16.1.10\" nat-destination-port=\"0\" elapsed-time=\"0\" inbound-bytes=\"0\" outbound-bytes=\"0\" inbound-packets=\"0\" outbound-packets=\"0\" source-zone-name=\"sec-zone-name-internet\" source-interface-name=\"reth0.11\" destination-zone-name=\"dst-sec-zone1-outside\" destination-interface-name=\"reth1.1\" packet-log-id=\"0\" alert=\"no\" username=\"N/A\" roles=\"N/A\" message=\"-\"", + "event.outcome": "success", + "event.severity": "165", + "event.start": "2017-10-12T19:55:55.792-02:00", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "DROP", + "juniper.srx.alert": "no", + "juniper.srx.application_name": "HTTP", + "juniper.srx.attack_name": "TROJAN:ZMEU-BOT-SCAN", + "juniper.srx.epoch_time": "1507845354", + "juniper.srx.export_id": "15229", + "juniper.srx.message_type": "SIG", + "juniper.srx.packet_log_id": "0", + "juniper.srx.policy_name": "Recommended", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.service_name": "SERVICE_IDP", + "juniper.srx.tag": "IDP_ATTACK_LOG_EVENT", + "juniper.srx.threat_severity": "HIGH", + "log.level": "notification", + "log.offset": 2773, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth1.1", + "observer.egress.zone": "dst-sec-zone1-outside", + "observer.ingress.interface.name": "reth0.11", + "observer.ingress.zone": "sec-zone-name-internet", + "observer.name": "idp1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "183.78.180.27", + "118.127.30.11", + "0.0.0.0", + "172.16.1.10" + ], + "rule.id": "9", + "rule.name": "IPS", + "server.bytes": 0, + "server.ip": "118.127.30.11", + "server.nat.port": 0, + "server.packets": 0, + "server.port": 80, + "service.type": "juniper", + "source.bytes": 0, + "source.ip": "183.78.180.27", + "source.nat.ip": "0.0.0.0", + "source.nat.port": 0, + "source.packets": 0, + "source.port": 45610, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2011-10-23T02:06:26.544-02:00", + "destination.ip": "172.27.14.203", + "destination.port": 80, + "event.action": "application_ddos", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1319367986\" ddos-application-name=\"Webserver\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.0\" destination-address=\"172.27.14.203\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" rulebase-name=\"DDOS\" policy-name=\"A DoS-Webserver\" repeat-count=\"0\" message=\"Connection rate exceeded limit 60\" context-value=\"N/A\"", + "event.outcome": "success", + "event.severity": "165", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.ddos_application_name": "Webserver", + "juniper.srx.epoch_time": "1319367986", + "juniper.srx.policy_name": "A DoS-Webserver", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.service_name": "HTTP", + "juniper.srx.tag": "IDP_APPDDOS_APP_STATE_EVENT", + "log.level": "notification", + "log.offset": 3693, + "message": "Connection rate exceeded limit 60", + "network.protocol": "TCP", + "observer.egress.interface.name": "reth0.0", + "observer.egress.zone": "untrust", + "observer.name": "SRX34001", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "172.27.14.203" + ], + "rule.id": "1", + "rule.name": "DDOS", + "server.ip": "172.27.14.203", + "server.port": 80, + "service.type": "juniper", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2011-10-23T16:28:31.696-02:00", + "client.ip": "192.168.14.214", + "client.port": 50825, + "destination.ip": "172.27.14.203", + "destination.port": 80, + "event.action": "application_ddos", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1319419711\" ddos-application-name=\"Webserver\" source-zone-name=\"trust\" source-interface-name=\"reth1.O\" source-address=\"192.168.14.214\" source-port=\"50825\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.0\" destination-address=\"172.27.14.203\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" ruleebase-name=\"DDOS\" policy-name=\"AppDoS-Webserver\" repeat-count=\"0\" action=\"NONE\" threat-severity=\"INFO\" connection-hit-rate=\"30\" context-name=\"http-get-url\" context-hit-rate=\"123\" context-value-hit-rate=\"0\" time-scope=\"PEER\" time-count=\"3\" time-period=\"60\" context-value=\"N/A\"", + "event.outcome": "success", + "event.severity": "165", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "NONE", + "juniper.srx.connection_hit_rate": "30", + "juniper.srx.context_hit_rate": "123", + "juniper.srx.context_name": "http-get-url", + "juniper.srx.context_value_hit_rate": "0", + "juniper.srx.ddos_application_name": "Webserver", + "juniper.srx.epoch_time": "1319419711", + "juniper.srx.policy_name": "AppDoS-Webserver", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.ruleebase_name": "DDOS", + "juniper.srx.service_name": "HTTP", + "juniper.srx.tag": "IDP_APPDDOS_APP_ATTACK_EVENT", + "juniper.srx.threat_severity": "INFO", + "juniper.srx.time_count": "3", + "juniper.srx.time_period": "60", + "juniper.srx.time_scope": "PEER", + "log.level": "notification", + "log.offset": 4165, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth0.0", + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "reth1.O", + "observer.ingress.zone": "trust", + "observer.name": "SRX34001", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.168.14.214", + "172.27.14.203" + ], + "rule.id": "1", + "server.ip": "172.27.14.203", + "server.port": 80, + "service.type": "juniper", + "source.ip": "192.168.14.214", + "source.port": 50825, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2012-10-23T17:28:31.696-02:00", + "client.ip": "193.168.14.214", + "client.port": 50825, + "destination.ip": "172.30.20.201", + "destination.port": 80, + "event.action": "application_ddos", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "epoch-time=\"1419419711\" ddos-application-name=\"Webserver\" source-zone-name=\"trust\" source-interface-name=\"reth3.0\" source-address=\"193.168.14.214\" source-port=\"50825\" destination-zone-name=\"untrust\" destination-interface-name=\"reth0.1\" destination-address=\"172.30.20.201\" destination-port=\"80\" protocol-name=\"TCP\" service-name=\"HTTP\" rule-name=\"1\" ruleebase-name=\"DDOS02\" policy-name=\"AppDoS-Webserver\" repeat-count=\"0\" action=\"NONE\" threat-severity=\"INFO\" connection-hit-rate=\"30\" context-name=\"http-get-url\" context-hit-rate=\"123\" context-value-hit-rate=\"0\" time-scope=\"PEER\" time-count=\"3\" time-period=\"60\" context-value=\"N/A\"", + "event.outcome": "success", + "event.severity": "165", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "NONE", + "juniper.srx.connection_hit_rate": "30", + "juniper.srx.context_hit_rate": "123", + "juniper.srx.context_name": "http-get-url", + "juniper.srx.context_value_hit_rate": "0", + "juniper.srx.ddos_application_name": "Webserver", + "juniper.srx.epoch_time": "1419419711", + "juniper.srx.policy_name": "AppDoS-Webserver", + "juniper.srx.process": "RT_IDP", + "juniper.srx.repeat_count": "0", + "juniper.srx.ruleebase_name": "DDOS02", + "juniper.srx.service_name": "HTTP", + "juniper.srx.tag": "IDP_APPDDOS_APP_ATTACK_EVENT_LS", + "juniper.srx.threat_severity": "INFO", + "juniper.srx.time_count": "3", + "juniper.srx.time_period": "60", + "juniper.srx.time_scope": "PEER", + "log.level": "notification", + "log.offset": 4895, + "network.protocol": "TCP", + "observer.egress.interface.name": "reth0.1", + "observer.egress.zone": "untrust", + "observer.ingress.interface.name": "reth3.0", + "observer.ingress.zone": "trust", + "observer.name": "SRX34001", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "193.168.14.214", + "172.30.20.201" + ], + "rule.id": "1", + "server.ip": "172.30.20.201", + "server.port": 80, + "service.type": "juniper", + "source.ip": "193.168.14.214", + "source.port": 50825, + "tags": [ + "juniper.srx", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/juniper/srx/test/ids.log b/x-pack/filebeat/module/juniper/srx/test/ids.log new file mode 100644 index 00000000000..5b87817da86 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/ids.log @@ -0,0 +1,12 @@ +<11>1 2018-07-19T18:17:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_TCP [junos@2636.1.1.1.2.137 attack-name="TCP sweep!" source-address="113.113.17.17" source-port="6000" destination-address="40.177.177.1" destination-port="1433" source-zone-name="untrust" interface-name="fe-0/0/2.0" action="drop"] +<11>1 2018-07-19T18:18:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_TCP [junos@2636.1.1.1.2.36 attack-name="WinNuke attack!" source-address="2000:0000:0000:0000:0000:0000:0000:0002" source-port="3240" destination-address="2001:0000:0000:0000:0000:0000:0000:0002" destination-port="139" source-zone-name="untrust" interface-name="fe-0/0/2.0" action="drop"] +<11>1 2018-07-19T18:19:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_TCP [junos@2636.1.1.1.2.40 attack-name="SYN flood!" source-address="1.1.1.2" source-port="40001" destination-address="2.2.2.2" destination-port="50010" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T18:22:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_UDP [junos@2636.1.1.1.2.40 attack-name="UDP flood!" source-address="111.1.1.3" source-port="40001" destination-address="3.4.2.2" destination-port="53" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T18:25:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_ICMP [junos@2636.1.1.1.2.40 attack-name="ICMP fragment!" source-address="111.1.1.3" destination-address="3.4.2.2" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T18:26:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_IP [junos@2636.1.1.1.2.40 attack-name="Record Route IP option!" source-address="111.1.1.3" destination-address="3.4.2.2" protocol-id="1" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T18:27:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_IP [junos@2636.1.1.1.2.40 attack-name="Tunnel GRE 6in6!" source-address="1212::12" destination-address="1111::11" protocol-id="1" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T18:28:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_IP [junos@2636.1.1.1.2.40 attack-name="Tunnel GRE 4in4!" source-address="12.12.12.1" destination-address="11.11.11.1" protocol-id="1" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2018-07-19T19:19:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_TCP_DST_IP [junos@2636.1.1.1.2.40 attack-name="SYN flood!" destination-address="2.2.2.2" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="alarm-without-drop"] +<11>1 2018-07-19T19:19:02.309-05:00 rtr199 RT_IDS - RT_SCREEN_TCP_SRC_IP [junos@2636.1.1.1.2.40 attack-name="SYN flood!" source-address="111.1.1.3" source-zone-name="trustZone" interface-name="ge-0/0/1.0" action="alarm-without-drop"] +<11>1 2020-07-17T09:54:43.912+02:00 rtr199 RT_IDS - RT_SCREEN_TCP [junos@2636.1.1.1.2.129 attack-name="TCP port scan!" source-address="10.1.1.100" source-port="50630" destination-address="10.1.1.1" destination-port="10778" source-zone-name="trust" interface-name="ge-0/0/1.0" action="drop"] +<11>1 2020-07-17T10:01:43.006+02:00 rtr199 RT_IDS - RT_SCREEN_TCP [junos@2636.1.1.1.2.129 attack-name="FIN but no ACK bit!" source-address="10.1.1.100" source-port="42799" destination-address="10.1.1.1" destination-port="7" source-zone-name="trust" interface-name="ge-0/0/1.0" action="drop"] diff --git a/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json new file mode 100644 index 00000000000..10abae2fa6d --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/ids.log-expected.json @@ -0,0 +1,699 @@ +[ + { + "@timestamp": "2018-07-19T21:17:02.309-02:00", + "client.ip": "113.113.17.17", + "client.port": 6000, + "destination.as.number": 4249, + "destination.as.organization.name": "Eli Lilly and Company", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "40.177.177.1", + "destination.port": 1433, + "event.action": "sweep_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"TCP sweep!\" source-address=\"113.113.17.17\" source-port=\"6000\" destination-address=\"40.177.177.1\" destination-port=\"1433\" source-zone-name=\"untrust\" interface-name=\"fe-0/0/2.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "TCP sweep!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP", + "log.level": "error", + "log.offset": 0, + "observer.ingress.interface.name": "fe-0/0/2.0", + "observer.ingress.zone": "untrust", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "113.113.17.17", + "40.177.177.1" + ], + "server.ip": "40.177.177.1", + "server.port": 1433, + "service.type": "juniper", + "source.as.number": 4134, + "source.as.organization.name": "No.31,Jin-rong Street", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.country_name": "China", + "source.geo.location.lat": 23.1167, + "source.geo.location.lon": 113.25, + "source.geo.region_iso_code": "CN-GD", + "source.geo.region_name": "Guangdong", + "source.ip": "113.113.17.17", + "source.port": 6000, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:18:02.309-02:00", + "client.ip": "2000:0000:0000:0000:0000:0000:0000:0002", + "client.port": 3240, + "destination.ip": "2001:0000:0000:0000:0000:0000:0000:0002", + "destination.port": 139, + "event.action": "attack_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"WinNuke attack!\" source-address=\"2000:0000:0000:0000:0000:0000:0000:0002\" source-port=\"3240\" destination-address=\"2001:0000:0000:0000:0000:0000:0000:0002\" destination-port=\"139\" source-zone-name=\"untrust\" interface-name=\"fe-0/0/2.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "WinNuke attack!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP", + "log.level": "error", + "log.offset": 294, + "observer.ingress.interface.name": "fe-0/0/2.0", + "observer.ingress.zone": "untrust", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "2000:0000:0000:0000:0000:0000:0000:0002", + "2001:0000:0000:0000:0000:0000:0000:0002" + ], + "server.ip": "2001:0000:0000:0000:0000:0000:0000:0002", + "server.port": 139, + "service.type": "juniper", + "source.ip": "2000:0000:0000:0000:0000:0000:0000:0002", + "source.port": 3240, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:19:02.309-02:00", + "client.ip": "1.1.1.2", + "client.port": 40001, + "destination.as.number": 3215, + "destination.as.organization.name": "Orange", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "FR", + "destination.geo.country_name": "France", + "destination.geo.location.lat": 48.8582, + "destination.geo.location.lon": 2.3387, + "destination.ip": "2.2.2.2", + "destination.port": 50010, + "event.action": "flood_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"SYN flood!\" source-address=\"1.1.1.2\" source-port=\"40001\" destination-address=\"2.2.2.2\" destination-port=\"50010\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "SYN flood!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP", + "log.level": "error", + "log.offset": 644, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "1.1.1.2", + "2.2.2.2" + ], + "server.ip": "2.2.2.2", + "server.port": 50010, + "service.type": "juniper", + "source.as.number": 13335, + "source.as.organization.name": "Cloudflare, Inc.", + "source.geo.continent_name": "Oceania", + "source.geo.country_iso_code": "AU", + "source.geo.country_name": "Australia", + "source.geo.location.lat": -33.494, + "source.geo.location.lon": 143.2104, + "source.ip": "1.1.1.2", + "source.port": 40001, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:22:02.309-02:00", + "client.ip": "111.1.1.3", + "client.port": 40001, + "destination.geo.city_name": "Seattle", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 47.6348, + "destination.geo.location.lon": -122.3451, + "destination.geo.region_iso_code": "US-WA", + "destination.geo.region_name": "Washington", + "destination.ip": "3.4.2.2", + "destination.port": 53, + "event.action": "flood_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"UDP flood!\" source-address=\"111.1.1.3\" source-port=\"40001\" destination-address=\"3.4.2.2\" destination-port=\"53\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "UDP flood!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_UDP", + "log.level": "error", + "log.offset": 930, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "111.1.1.3", + "3.4.2.2" + ], + "server.ip": "3.4.2.2", + "server.port": 53, + "service.type": "juniper", + "source.as.number": 56041, + "source.as.organization.name": "China Mobile communications corporation", + "source.geo.city_name": "Wenzhou", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.country_name": "China", + "source.geo.location.lat": 27.9983, + "source.geo.location.lon": 120.6666, + "source.geo.region_iso_code": "CN-ZJ", + "source.geo.region_name": "Zhejiang", + "source.ip": "111.1.1.3", + "source.port": 40001, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:25:02.309-02:00", + "client.ip": "111.1.1.3", + "destination.geo.city_name": "Seattle", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 47.6348, + "destination.geo.location.lon": -122.3451, + "destination.geo.region_iso_code": "US-WA", + "destination.geo.region_name": "Washington", + "destination.ip": "3.4.2.2", + "event.action": "fragment_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"ICMP fragment!\" source-address=\"111.1.1.3\" destination-address=\"3.4.2.2\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "ICMP fragment!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_ICMP", + "log.level": "error", + "log.offset": 1215, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "111.1.1.3", + "3.4.2.2" + ], + "server.ip": "3.4.2.2", + "service.type": "juniper", + "source.as.number": 56041, + "source.as.organization.name": "China Mobile communications corporation", + "source.geo.city_name": "Wenzhou", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.country_name": "China", + "source.geo.location.lat": 27.9983, + "source.geo.location.lon": 120.6666, + "source.geo.region_iso_code": "CN-ZJ", + "source.geo.region_name": "Zhejiang", + "source.ip": "111.1.1.3", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:26:02.309-02:00", + "client.ip": "111.1.1.3", + "destination.geo.city_name": "Seattle", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 47.6348, + "destination.geo.location.lon": -122.3451, + "destination.geo.region_iso_code": "US-WA", + "destination.geo.region_name": "Washington", + "destination.ip": "3.4.2.2", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"Record Route IP option!\" source-address=\"111.1.1.3\" destination-address=\"3.4.2.2\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "Record Route IP option!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_IP", + "log.level": "error", + "log.offset": 1463, + "network.iana_number": "1", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "111.1.1.3", + "3.4.2.2" + ], + "server.ip": "3.4.2.2", + "service.type": "juniper", + "source.as.number": 56041, + "source.as.organization.name": "China Mobile communications corporation", + "source.geo.city_name": "Wenzhou", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.country_name": "China", + "source.geo.location.lat": 27.9983, + "source.geo.location.lon": 120.6666, + "source.geo.region_iso_code": "CN-ZJ", + "source.geo.region_name": "Zhejiang", + "source.ip": "111.1.1.3", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:27:02.309-02:00", + "client.ip": "1212::12", + "destination.ip": "1111::11", + "event.action": "tunneling_screen", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"Tunnel GRE 6in6!\" source-address=\"1212::12\" destination-address=\"1111::11\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "Tunnel GRE 6in6!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_IP", + "log.level": "error", + "log.offset": 1734, + "network.iana_number": "1", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "1212::12", + "1111::11" + ], + "server.ip": "1111::11", + "service.type": "juniper", + "source.ip": "1212::12", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T21:28:02.309-02:00", + "client.ip": "12.12.12.1", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "11.11.11.1", + "event.action": "tunneling_screen", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"Tunnel GRE 4in4!\" source-address=\"12.12.12.1\" destination-address=\"11.11.11.1\" protocol-id=\"1\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "Tunnel GRE 4in4!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_IP", + "log.level": "error", + "log.offset": 1998, + "network.iana_number": "1", + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "12.12.12.1", + "11.11.11.1" + ], + "server.ip": "11.11.11.1", + "service.type": "juniper", + "source.as.number": 32328, + "source.as.organization.name": "Alascom, Inc.", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "12.12.12.1", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T22:19:02.309-02:00", + "destination.as.number": 3215, + "destination.as.organization.name": "Orange", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "FR", + "destination.geo.country_name": "France", + "destination.geo.location.lat": 48.8582, + "destination.geo.location.lon": 2.3387, + "destination.ip": "2.2.2.2", + "event.action": "flood_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"SYN flood!\" destination-address=\"2.2.2.2\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"alarm-without-drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "alarm-without-drop", + "juniper.srx.attack_name": "SYN flood!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP_DST_IP", + "log.level": "error", + "log.offset": 2266, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "2.2.2.2" + ], + "server.ip": "2.2.2.2", + "service.type": "juniper", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2018-07-19T22:19:02.309-02:00", + "client.ip": "111.1.1.3", + "event.action": "flood_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"SYN flood!\" source-address=\"111.1.1.3\" source-zone-name=\"trustZone\" interface-name=\"ge-0/0/1.0\" action=\"alarm-without-drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "alarm-without-drop", + "juniper.srx.attack_name": "SYN flood!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP_SRC_IP", + "log.level": "error", + "log.offset": 2503, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trustZone", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "111.1.1.3" + ], + "service.type": "juniper", + "source.as.number": 56041, + "source.as.organization.name": "China Mobile communications corporation", + "source.geo.city_name": "Wenzhou", + "source.geo.continent_name": "Asia", + "source.geo.country_iso_code": "CN", + "source.geo.country_name": "China", + "source.geo.location.lat": 27.9983, + "source.geo.location.lon": 120.6666, + "source.geo.region_iso_code": "CN-ZJ", + "source.geo.region_name": "Zhejiang", + "source.ip": "111.1.1.3", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-17T05:54:43.912-02:00", + "client.ip": "10.1.1.100", + "client.port": 50630, + "destination.ip": "10.1.1.1", + "destination.port": 10778, + "event.action": "scan_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"TCP port scan!\" source-address=\"10.1.1.100\" source-port=\"50630\" destination-address=\"10.1.1.1\" destination-port=\"10778\" source-zone-name=\"trust\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "TCP port scan!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP", + "log.level": "error", + "log.offset": 2737, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trust", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "10.1.1.1" + ], + "server.ip": "10.1.1.1", + "server.port": 10778, + "service.type": "juniper", + "source.ip": "10.1.1.100", + "source.port": 50630, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2020-07-17T06:01:43.006-02:00", + "client.ip": "10.1.1.100", + "client.port": 42799, + "destination.ip": "10.1.1.1", + "destination.port": 7, + "event.action": "illegal_tcp_flag_detected", + "event.category": [ + "network", + "intrusion_detection" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "attack-name=\"FIN but no ACK bit!\" source-address=\"10.1.1.100\" source-port=\"42799\" destination-address=\"10.1.1.1\" destination-port=\"7\" source-zone-name=\"trust\" interface-name=\"ge-0/0/1.0\" action=\"drop\"", + "event.outcome": "success", + "event.severity": "11", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.attack_name": "FIN but no ACK bit!", + "juniper.srx.process": "RT_IDS", + "juniper.srx.tag": "RT_SCREEN_TCP", + "log.level": "error", + "log.offset": 3028, + "observer.ingress.interface.name": "ge-0/0/1.0", + "observer.ingress.zone": "trust", + "observer.name": "rtr199", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.1.1.100", + "10.1.1.1" + ], + "server.ip": "10.1.1.1", + "server.port": 7, + "service.type": "juniper", + "source.ip": "10.1.1.100", + "source.port": 42799, + "tags": [ + "juniper.srx", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/juniper/srx/test/secintel.log b/x-pack/filebeat/module/juniper/srx/test/secintel.log new file mode 100644 index 00000000000..12f8f137c7f --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/secintel.log @@ -0,0 +1,2 @@ +<14>1 2016-10-17T15:18:11.618Z SRX-1500 RT_SECINTEL - SECINTEL_ACTION_LOG [junos@2636.1.1.1.2.129 category="secintel" sub-category="Blacklist" action="BLOCK" action-detail="DROP" http-host="N/A" threat-severity="0" source-address="5.196.121.161" source-port="1" destination-address="10.10.0.10" destination-port="24039" protocol-id="1" application="N/A" nested-application="N/A" feed-name="Tor_Exit_Nodes" policy-name="cc_policy" profile-name="Blacklist" username="N/A" roles="N/A" session-id-32="572564" source-zone-name="Outside" destination-zone-name="DMZ"] +<14>1 2016-10-17T15:18:11.618Z SRX-1500 RT_SECINTEL - SECINTEL_ACTION_LOG [junos@2636.1.1.1.2.129 category="secintel" sub-category="CC" action="BLOCK" action-detail="CLOSE REDIRECT MSG" http-host="dummy_host" threat-severity="10" source-address="1.1.1.1" source-port="36612" destination-address="10.0.0.1" destination-port="80" protocol-id="6" application="HTTP" nested-application="N/A" feed-name="cc_url_data" policy-name="test" profile-name="test-profile" username="N/A" roles="N/A" session-id-32="502362" source-zone-name="Inside" destination-zone-name="Outside" occur-count="0"] diff --git a/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json new file mode 100644 index 00000000000..49667e85897 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/secintel.log-expected.json @@ -0,0 +1,140 @@ +[ + { + "@timestamp": "2016-10-17T13:18:11.618-02:00", + "client.ip": "5.196.121.161", + "client.port": 1, + "destination.ip": "10.10.0.10", + "destination.port": 24039, + "event.action": "malware_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "category=\"secintel\" sub-category=\"Blacklist\" action=\"BLOCK\" action-detail=\"DROP\" http-host=\"N/A\" threat-severity=\"0\" source-address=\"5.196.121.161\" source-port=\"1\" destination-address=\"10.10.0.10\" destination-port=\"24039\" protocol-id=\"1\" application=\"N/A\" nested-application=\"N/A\" feed-name=\"Tor_Exit_Nodes\" policy-name=\"cc_policy\" profile-name=\"Blacklist\" username=\"N/A\" roles=\"N/A\" session-id-32=\"572564\" source-zone-name=\"Outside\" destination-zone-name=\"DMZ\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "BLOCK", + "juniper.srx.action_detail": "DROP", + "juniper.srx.category": "secintel", + "juniper.srx.feed_name": "Tor_Exit_Nodes", + "juniper.srx.policy_name": "cc_policy", + "juniper.srx.process": "RT_SECINTEL", + "juniper.srx.profile_name": "Blacklist", + "juniper.srx.session_id_32": "572564", + "juniper.srx.sub_category": "Blacklist", + "juniper.srx.tag": "SECINTEL_ACTION_LOG", + "juniper.srx.threat_severity": "0", + "log.level": "informational", + "log.offset": 0, + "network.iana_number": "1", + "observer.egress.zone": "DMZ", + "observer.ingress.zone": "Outside", + "observer.name": "SRX-1500", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "5.196.121.161", + "10.10.0.10" + ], + "server.ip": "10.10.0.10", + "server.port": 24039, + "service.type": "juniper", + "source.as.number": 16276, + "source.as.organization.name": "OVH SAS", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "FR", + "source.geo.country_name": "France", + "source.geo.location.lat": 48.8582, + "source.geo.location.lon": 2.3387, + "source.ip": "5.196.121.161", + "source.port": 1, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2016-10-17T13:18:11.618-02:00", + "client.ip": "1.1.1.1", + "client.port": 36612, + "destination.ip": "10.0.0.1", + "destination.port": 80, + "event.action": "malware_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "category=\"secintel\" sub-category=\"CC\" action=\"BLOCK\" action-detail=\"CLOSE REDIRECT MSG\" http-host=\"dummy_host\" threat-severity=\"10\" source-address=\"1.1.1.1\" source-port=\"36612\" destination-address=\"10.0.0.1\" destination-port=\"80\" protocol-id=\"6\" application=\"HTTP\" nested-application=\"N/A\" feed-name=\"cc_url_data\" policy-name=\"test\" profile-name=\"test-profile\" username=\"N/A\" roles=\"N/A\" session-id-32=\"502362\" source-zone-name=\"Inside\" destination-zone-name=\"Outside\" occur-count=\"0\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "BLOCK", + "juniper.srx.action_detail": "CLOSE REDIRECT MSG", + "juniper.srx.application": "HTTP", + "juniper.srx.category": "secintel", + "juniper.srx.feed_name": "cc_url_data", + "juniper.srx.occur_count": "0", + "juniper.srx.policy_name": "test", + "juniper.srx.process": "RT_SECINTEL", + "juniper.srx.profile_name": "test-profile", + "juniper.srx.session_id_32": "502362", + "juniper.srx.sub_category": "CC", + "juniper.srx.tag": "SECINTEL_ACTION_LOG", + "juniper.srx.threat_severity": "10", + "log.level": "informational", + "log.offset": 561, + "network.iana_number": "6", + "observer.egress.zone": "Outside", + "observer.ingress.zone": "Inside", + "observer.name": "SRX-1500", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "dummy_host" + ], + "related.ip": [ + "1.1.1.1", + "10.0.0.1" + ], + "server.ip": "10.0.0.1", + "server.port": 80, + "service.type": "juniper", + "source.as.number": 13335, + "source.as.organization.name": "Cloudflare, Inc.", + "source.geo.continent_name": "Oceania", + "source.geo.country_iso_code": "AU", + "source.geo.country_name": "Australia", + "source.geo.location.lat": -33.494, + "source.geo.location.lon": 143.2104, + "source.ip": "1.1.1.1", + "source.port": 36612, + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "dummy_host" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/juniper/srx/test/utm.log b/x-pack/filebeat/module/juniper/srx/test/utm.log new file mode 100644 index 00000000000..61c320ae885 --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/utm.log @@ -0,0 +1,12 @@ +<12>1 2016-02-18T01:32:50.391Z utm-srx550-b RT_UTM - WEBFILTER_URL_BLOCKED [junos@2636.1.1.1.2.86 source-address="192.168.1.100" source-port="58071" destination-address="103.235.46.39" destination-port="80" category="cat1" reason="BY_BLACK_LIST" profile="uf1" url="www.baidu.com" obj="/" username="user01" roles="N/A"] +<12>1 2016-02-18T01:32:50.391Z utm-srx550-b RT_UTM - WEBFILTER_URL_PERMITTED [junos@2636.1.1.1.2.86 source-address="10.10.10.50" source-port="1402" destination-address="216.200.241.66" destination-port="80" category="N/A" reason="BY_OTHER" profile="wf-profile" url="www.checkpoint.com" obj="/css/homepage2012.css" username="user02" roles="N/A"] +<12>1 2010-02-08T08:29:28.565Z SRX650-1 RT_UTM - AV_VIRUS_DETECTED_MT [junos@2636.1.1.1.2.40 source-address="188.40.238.250" source-port="80" destination-address="10.1.1.103" destination-port="47095" source-zone-name="untrust" filename="www.eicar.org/download/eicar.com" temporary-filename="www.eicar.org/download/eicar.com" name="EICAR-Test-File" url="EICAR-Test-File"] +<12>1 2010-02-08T08:29:28.565Z SRX650-1 RT_UTM - AV_SCANNER_DROP_FILE_MT [junos@2636.1.1.1.2.40 source-address="74.125.155.147" source-port="80" destination-address="10.1.1.103" destination-port="33578" filename="www.google.com/" error-code="14" error-message="scan engine is not ready"] +<12>1 2010-01-29T10:59:59.660Z SRX650-1 RT_UTM - AV_HUGE_FILE_DROPPED_MT [junos@2636.1.1.1.2.40 source-address="10.2.1.101" source-port="80" destination-address="10.1.1.103" destination-port="51727" filename="10.2.1.101/images/junos- srxsme-10.2-20100106.0-domestic.tgz"] +<14>1 2016-02-18T01:33:50.391Z utm-srx550-b RT_UTM - ANTISPAM_SPAM_DETECTED_MT [junos@2636.1.1.1.2.86 source-zone="trust" destination-zone="untrust" source-name="N/A" source-address="10.10.10.1" profile-name="antispam01" action="drop" reason="Match local blacklist" username="user01" roles="N/A"] +<14>1 2016-02-18T01:34:50.391Z utm-srx550-b RT_UTM - CONTENT_FILTERING_BLOCKED_MT [junos@2636.1.1.1.2.86 source-zone="untrust" destination-zone="trust" protocol="http" source-address="192.0.2.3" source-port="58071" destination-address="198.51.100.2" destination-port="80" profile-name="content02" action="drop" reason="blocked due to file extension block list" username="user01@testuser.com" roles="N/A" filename="test.cmd"] +<12>1 2016-02-19T01:32:50.391Z utm-srx550-b RT_UTM - WEBFILTER_URL_BLOCKED_LS [junos@2636.1.1.1.2.86 source-address="192.168.1.100" source-port="58071" destination-address="103.235.46.39" destination-port="80" category="cat1" reason="BY_BLACK_LIST" profile="uf1" url="www.baidu.com" obj="/" username="user01" roles="N/A"] +<12>1 2011-02-08T08:29:28.565Z SRX650-1 RT_UTM - AV_VIRUS_DETECTED_MT_LS [junos@2636.1.1.1.2.40 source-address="188.40.238.250" source-port="80" destination-address="10.1.1.103" destination-port="47095" source-zone-name="untrust" filename="www.eicar.org/download/eicar.com" temporary-filename="www.eicar.org/download/eicar.com" name="EICAR-Test-File" url="EICAR-Test-File"] +<14>1 2020-07-14T14:16:18.345Z SRX650-1 RT_UTM - WEBFILTER_URL_PERMITTED [junos@2636.1.1.1.2.129 source-zone="trust" destination-zone="untrust" source-address="10.1.1.100" source-port="58974" destination-address="104.26.15.142" destination-port="443" session-id="16297" application="UNKNOWN" nested-application="UNKNOWN" category="Enhanced_Information_Technology" reason="BY_SITE_REPUTATION_MODERATELY_SAFE" profile="WCF1" url="datawrapper.dwcdn.net" obj="/" username="N/A" roles="N/A" application-sub-category="N/A" urlcategory-risk="0"] +<12>1 2020-07-14T14:16:29.541Z SRX650-1 RT_UTM - WEBFILTER_URL_BLOCKED [junos@2636.1.1.1.2.129 source-zone="trust" destination-zone="untrust" source-address="10.1.1.100" source-port="59075" destination-address="85.114.159.93" destination-port="443" session-id="16490" application="UNKNOWN" nested-application="UNKNOWN" category="Enhanced_Advertisements" reason="BY_SITE_REPUTATION_SUSPICIOUS" profile="WCF1" url="dsp.adfarm1.adition.com" obj="/" username="N/A" roles="N/A" application-sub-category="N/A" urlcategory-risk="3"] +<12>1 2020-07-14T14:17:04.733Z SRX650-1 RT_UTM - AV_FILE_NOT_SCANNED_DROPPED_MT [junos@2636.1.1.1.2.129 source-zone="trust" destination-zone="untrust" source-address="23.209.86.45" source-port="80" destination-address="10.1.1.100" destination-port="58954" profile-name="Custom-Sophos-Profile" filename="download.cdn.mozilla.net/pub/firefox/releases/78.0.2/update/win64/de/firefox-78.0.2.complete.mar" action="BLOCKED" reason="exceeding maximum content size" error-code="7" username="N/A" roles="N/A"] diff --git a/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json b/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json new file mode 100644 index 00000000000..f9890a6ca0f --- /dev/null +++ b/x-pack/filebeat/module/juniper/srx/test/utm.log-expected.json @@ -0,0 +1,698 @@ +[ + { + "@timestamp": "2016-02-17T23:32:50.391-02:00", + "client.ip": "192.168.1.100", + "client.port": 58071, + "destination.as.number": 55967, + "destination.as.organization.name": "Beijing Baidu Netcom Science and Technology Co., Ltd.", + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "HK", + "destination.geo.country_name": "Hong Kong", + "destination.geo.location.lat": 22.25, + "destination.geo.location.lon": 114.1667, + "destination.ip": "103.235.46.39", + "destination.port": 80, + "event.action": "web_filter", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-address=\"192.168.1.100\" source-port=\"58071\" destination-address=\"103.235.46.39\" destination-port=\"80\" category=\"cat1\" reason=\"BY_BLACK_LIST\" profile=\"uf1\" url=\"www.baidu.com\" obj=\"/\" username=\"user01\" roles=\"N/A\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.category": "cat1", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile": "uf1", + "juniper.srx.reason": "BY_BLACK_LIST", + "juniper.srx.tag": "WEBFILTER_URL_BLOCKED", + "log.level": "warning", + "log.offset": 0, + "observer.name": "utm-srx550-b", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "www.baidu.com" + ], + "related.ip": [ + "192.168.1.100", + "103.235.46.39" + ], + "server.ip": "103.235.46.39", + "server.port": 80, + "service.type": "juniper", + "source.ip": "192.168.1.100", + "source.port": 58071, + "source.user.name": "user01", + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "www.baidu.com", + "url.path": "/" + }, + { + "@timestamp": "2016-02-17T23:32:50.391-02:00", + "client.ip": "10.10.10.50", + "client.port": 1402, + "destination.as.number": 6461, + "destination.as.organization.name": "Zayo Bandwidth", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "216.200.241.66", + "destination.port": 80, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.10.10.50\" source-port=\"1402\" destination-address=\"216.200.241.66\" destination-port=\"80\" category=\"N/A\" reason=\"BY_OTHER\" profile=\"wf-profile\" url=\"www.checkpoint.com\" obj=\"/css/homepage2012.css\" username=\"user02\" roles=\"N/A\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile": "wf-profile", + "juniper.srx.reason": "BY_OTHER", + "juniper.srx.tag": "WEBFILTER_URL_PERMITTED", + "log.level": "warning", + "log.offset": 319, + "observer.name": "utm-srx550-b", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "www.checkpoint.com" + ], + "related.ip": [ + "10.10.10.50", + "216.200.241.66" + ], + "server.ip": "216.200.241.66", + "server.port": 80, + "service.type": "juniper", + "source.ip": "10.10.10.50", + "source.port": 1402, + "source.user.name": "user02", + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "www.checkpoint.com", + "url.path": "/css/homepage2012.css" + }, + { + "@timestamp": "2010-02-08T06:29:28.565-02:00", + "client.ip": "188.40.238.250", + "client.port": 80, + "destination.ip": "10.1.1.103", + "destination.port": 47095, + "event.action": "virus_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-address=\"188.40.238.250\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"47095\" source-zone-name=\"untrust\" filename=\"www.eicar.org/download/eicar.com\" temporary-filename=\"www.eicar.org/download/eicar.com\" name=\"EICAR-Test-File\" url=\"EICAR-Test-File\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "file.name": "www.eicar.org/download/eicar.com", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.name": "EICAR-Test-File", + "juniper.srx.process": "RT_UTM", + "juniper.srx.tag": "AV_VIRUS_DETECTED_MT", + "juniper.srx.temporary_filename": "www.eicar.org/download/eicar.com", + "log.level": "warning", + "log.offset": 664, + "observer.ingress.zone": "untrust", + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "EICAR-Test-File" + ], + "related.ip": [ + "188.40.238.250", + "10.1.1.103" + ], + "server.ip": "10.1.1.103", + "server.port": 47095, + "service.type": "juniper", + "source.as.number": 24940, + "source.as.organization.name": "Hetzner Online GmbH", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.country_name": "Germany", + "source.geo.location.lat": 51.2993, + "source.geo.location.lon": 9.491, + "source.ip": "188.40.238.250", + "source.port": 80, + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "EICAR-Test-File" + }, + { + "@timestamp": "2010-02-08T06:29:28.565-02:00", + "client.ip": "74.125.155.147", + "client.port": 80, + "destination.ip": "10.1.1.103", + "destination.port": 33578, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"74.125.155.147\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"33578\" filename=\"www.google.com/\" error-code=\"14\" error-message=\"scan engine is not ready\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "file.name": "www.google.com/", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.error_code": "14", + "juniper.srx.error_message": "scan engine is not ready", + "juniper.srx.process": "RT_UTM", + "juniper.srx.tag": "AV_SCANNER_DROP_FILE_MT", + "log.level": "warning", + "log.offset": 1035, + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "74.125.155.147", + "10.1.1.103" + ], + "server.ip": "10.1.1.103", + "server.port": 33578, + "service.type": "juniper", + "source.as.number": 15169, + "source.as.organization.name": "Google LLC", + "source.geo.continent_name": "North America", + "source.geo.country_iso_code": "US", + "source.geo.country_name": "United States", + "source.geo.location.lat": 37.751, + "source.geo.location.lon": -97.822, + "source.ip": "74.125.155.147", + "source.port": 80, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2010-01-29T08:59:59.660-02:00", + "client.ip": "10.2.1.101", + "client.port": 80, + "destination.ip": "10.1.1.103", + "destination.port": 51727, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-address=\"10.2.1.101\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"51727\" filename=\"10.2.1.101/images/junos- srxsme-10.2-20100106.0-domestic.tgz\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "file.name": "10.2.1.101/images/junos- srxsme-10.2-20100106.0-domestic.tgz", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.process": "RT_UTM", + "juniper.srx.tag": "AV_HUGE_FILE_DROPPED_MT", + "log.level": "warning", + "log.offset": 1323, + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.2.1.101", + "10.1.1.103" + ], + "server.ip": "10.1.1.103", + "server.port": 51727, + "service.type": "juniper", + "source.ip": "10.2.1.101", + "source.port": 80, + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2016-02-17T23:33:50.391-02:00", + "client.ip": "10.10.10.1", + "event.action": "antispam_filter", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-name=\"N/A\" source-address=\"10.10.10.1\" profile-name=\"antispam01\" action=\"drop\" reason=\"Match local blacklist\" username=\"user01\" roles=\"N/A\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile_name": "antispam01", + "juniper.srx.reason": "Match local blacklist", + "juniper.srx.tag": "ANTISPAM_SPAM_DETECTED_MT", + "log.level": "informational", + "log.offset": 1595, + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "utm-srx550-b", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "10.10.10.1" + ], + "service.type": "juniper", + "source.ip": "10.10.10.1", + "source.user.name": "user01", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2016-02-17T23:34:50.391-02:00", + "client.ip": "192.0.2.3", + "client.port": 58071, + "destination.ip": "198.51.100.2", + "destination.port": 80, + "event.action": "content_filter", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-zone=\"untrust\" destination-zone=\"trust\" protocol=\"http\" source-address=\"192.0.2.3\" source-port=\"58071\" destination-address=\"198.51.100.2\" destination-port=\"80\" profile-name=\"content02\" action=\"drop\" reason=\"blocked due to file extension block list\" username=\"user01@testuser.com\" roles=\"N/A\" filename=\"test.cmd\"", + "event.outcome": "success", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "file.name": "test.cmd", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "drop", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile_name": "content02", + "juniper.srx.reason": "blocked due to file extension block list", + "juniper.srx.tag": "CONTENT_FILTERING_BLOCKED_MT", + "log.level": "informational", + "log.offset": 1892, + "network.protocol": "http", + "observer.egress.zone": "trust", + "observer.ingress.zone": "untrust", + "observer.name": "utm-srx550-b", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "192.0.2.3", + "198.51.100.2" + ], + "server.ip": "198.51.100.2", + "server.port": 80, + "service.type": "juniper", + "source.ip": "192.0.2.3", + "source.port": 58071, + "source.user.name": "user01@testuser.com", + "tags": [ + "juniper.srx", + "forwarded" + ] + }, + { + "@timestamp": "2016-02-18T23:32:50.391-02:00", + "client.ip": "192.168.1.100", + "client.port": 58071, + "destination.as.number": 55967, + "destination.as.organization.name": "Beijing Baidu Netcom Science and Technology Co., Ltd.", + "destination.geo.continent_name": "Asia", + "destination.geo.country_iso_code": "HK", + "destination.geo.country_name": "Hong Kong", + "destination.geo.location.lat": 22.25, + "destination.geo.location.lon": 114.1667, + "destination.ip": "103.235.46.39", + "destination.port": 80, + "event.action": "web_filter", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-address=\"192.168.1.100\" source-port=\"58071\" destination-address=\"103.235.46.39\" destination-port=\"80\" category=\"cat1\" reason=\"BY_BLACK_LIST\" profile=\"uf1\" url=\"www.baidu.com\" obj=\"/\" username=\"user01\" roles=\"N/A\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.category": "cat1", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile": "uf1", + "juniper.srx.reason": "BY_BLACK_LIST", + "juniper.srx.tag": "WEBFILTER_URL_BLOCKED_LS", + "log.level": "warning", + "log.offset": 2317, + "observer.name": "utm-srx550-b", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "www.baidu.com" + ], + "related.ip": [ + "192.168.1.100", + "103.235.46.39" + ], + "server.ip": "103.235.46.39", + "server.port": 80, + "service.type": "juniper", + "source.ip": "192.168.1.100", + "source.port": 58071, + "source.user.name": "user01", + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "www.baidu.com", + "url.path": "/" + }, + { + "@timestamp": "2011-02-08T06:29:28.565-02:00", + "client.ip": "188.40.238.250", + "client.port": 80, + "destination.ip": "10.1.1.103", + "destination.port": 47095, + "event.action": "virus_detected", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-address=\"188.40.238.250\" source-port=\"80\" destination-address=\"10.1.1.103\" destination-port=\"47095\" source-zone-name=\"untrust\" filename=\"www.eicar.org/download/eicar.com\" temporary-filename=\"www.eicar.org/download/eicar.com\" name=\"EICAR-Test-File\" url=\"EICAR-Test-File\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "file.name": "www.eicar.org/download/eicar.com", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.name": "EICAR-Test-File", + "juniper.srx.process": "RT_UTM", + "juniper.srx.tag": "AV_VIRUS_DETECTED_MT_LS", + "juniper.srx.temporary_filename": "www.eicar.org/download/eicar.com", + "log.level": "warning", + "log.offset": 2639, + "observer.ingress.zone": "untrust", + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "EICAR-Test-File" + ], + "related.ip": [ + "188.40.238.250", + "10.1.1.103" + ], + "server.ip": "10.1.1.103", + "server.port": 47095, + "service.type": "juniper", + "source.as.number": 24940, + "source.as.organization.name": "Hetzner Online GmbH", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "DE", + "source.geo.country_name": "Germany", + "source.geo.location.lat": 51.2993, + "source.geo.location.lon": 9.491, + "source.ip": "188.40.238.250", + "source.port": 80, + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "EICAR-Test-File" + }, + { + "@timestamp": "2020-07-14T12:16:18.345-02:00", + "client.ip": "10.1.1.100", + "client.port": 58974, + "destination.as.number": 13335, + "destination.as.organization.name": "Cloudflare, Inc.", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.country_name": "United States", + "destination.geo.location.lat": 37.751, + "destination.geo.location.lon": -97.822, + "destination.ip": "104.26.15.142", + "destination.port": 443, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"10.1.1.100\" source-port=\"58974\" destination-address=\"104.26.15.142\" destination-port=\"443\" session-id=\"16297\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" category=\"Enhanced_Information_Technology\" reason=\"BY_SITE_REPUTATION_MODERATELY_SAFE\" profile=\"WCF1\" url=\"datawrapper.dwcdn.net\" obj=\"/\" username=\"N/A\" roles=\"N/A\" application-sub-category=\"N/A\" urlcategory-risk=\"0\"", + "event.outcome": "success", + "event.risk_score": "0", + "event.severity": "14", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.category": "Enhanced_Information_Technology", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile": "WCF1", + "juniper.srx.reason": "BY_SITE_REPUTATION_MODERATELY_SAFE", + "juniper.srx.session_id": "16297", + "juniper.srx.tag": "WEBFILTER_URL_PERMITTED", + "log.level": "informational", + "log.offset": 3013, + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "datawrapper.dwcdn.net" + ], + "related.ip": [ + "10.1.1.100", + "104.26.15.142" + ], + "server.ip": "104.26.15.142", + "server.port": 443, + "service.type": "juniper", + "source.ip": "10.1.1.100", + "source.port": 58974, + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "datawrapper.dwcdn.net", + "url.path": "/" + }, + { + "@timestamp": "2020-07-14T12:16:29.541-02:00", + "client.ip": "10.1.1.100", + "client.port": 59075, + "destination.as.number": 24961, + "destination.as.organization.name": "myLoc managed IT AG", + "destination.geo.continent_name": "Europe", + "destination.geo.country_iso_code": "DE", + "destination.geo.country_name": "Germany", + "destination.geo.location.lat": 51.2993, + "destination.geo.location.lon": 9.491, + "destination.ip": "85.114.159.93", + "destination.port": 443, + "event.action": "web_filter", + "event.category": [ + "network", + "malware" + ], + "event.dataset": "juniper.srx", + "event.kind": "alert", + "event.module": "juniper", + "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"10.1.1.100\" source-port=\"59075\" destination-address=\"85.114.159.93\" destination-port=\"443\" session-id=\"16490\" application=\"UNKNOWN\" nested-application=\"UNKNOWN\" category=\"Enhanced_Advertisements\" reason=\"BY_SITE_REPUTATION_SUSPICIOUS\" profile=\"WCF1\" url=\"dsp.adfarm1.adition.com\" obj=\"/\" username=\"N/A\" roles=\"N/A\" application-sub-category=\"N/A\" urlcategory-risk=\"3\"", + "event.outcome": "success", + "event.risk_score": "3", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "info", + "denied", + "connection" + ], + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.category": "Enhanced_Advertisements", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile": "WCF1", + "juniper.srx.reason": "BY_SITE_REPUTATION_SUSPICIOUS", + "juniper.srx.session_id": "16490", + "juniper.srx.tag": "WEBFILTER_URL_BLOCKED", + "log.level": "warning", + "log.offset": 3552, + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.hosts": [ + "dsp.adfarm1.adition.com" + ], + "related.ip": [ + "10.1.1.100", + "85.114.159.93" + ], + "server.ip": "85.114.159.93", + "server.port": 443, + "service.type": "juniper", + "source.ip": "10.1.1.100", + "source.port": 59075, + "tags": [ + "juniper.srx", + "forwarded" + ], + "url.domain": "dsp.adfarm1.adition.com", + "url.path": "/" + }, + { + "@timestamp": "2020-07-14T12:17:04.733-02:00", + "client.ip": "23.209.86.45", + "client.port": 80, + "destination.ip": "10.1.1.100", + "destination.port": 58954, + "event.category": [ + "network" + ], + "event.dataset": "juniper.srx", + "event.kind": "event", + "event.module": "juniper", + "event.original": "source-zone=\"trust\" destination-zone=\"untrust\" source-address=\"23.209.86.45\" source-port=\"80\" destination-address=\"10.1.1.100\" destination-port=\"58954\" profile-name=\"Custom-Sophos-Profile\" filename=\"download.cdn.mozilla.net/pub/firefox/releases/78.0.2/update/win64/de/firefox-78.0.2.complete.mar\" action=\"BLOCKED\" reason=\"exceeding maximum content size\" error-code=\"7\" username=\"N/A\" roles=\"N/A\"", + "event.outcome": "success", + "event.severity": "12", + "event.timezone": "-02:00", + "event.type": [ + "allowed", + "connection" + ], + "file.name": "download.cdn.mozilla.net/pub/firefox/releases/78.0.2/update/win64/de/firefox-78.0.2.complete.mar", + "fileset.name": "srx", + "input.type": "log", + "juniper.srx.action": "BLOCKED", + "juniper.srx.error_code": "7", + "juniper.srx.process": "RT_UTM", + "juniper.srx.profile_name": "Custom-Sophos-Profile", + "juniper.srx.reason": "exceeding maximum content size", + "juniper.srx.tag": "AV_FILE_NOT_SCANNED_DROPPED_MT", + "log.level": "warning", + "log.offset": 4078, + "observer.egress.zone": "untrust", + "observer.ingress.zone": "trust", + "observer.name": "SRX650-1", + "observer.product": "SRX", + "observer.type": "firewall", + "observer.vendor": "Juniper", + "related.ip": [ + "23.209.86.45", + "10.1.1.100" + ], + "server.ip": "10.1.1.100", + "server.port": 58954, + "service.type": "juniper", + "source.as.number": 16625, + "source.as.organization.name": "Akamai Technologies, Inc.", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "NL", + "source.geo.country_name": "Netherlands", + "source.geo.location.lat": 52.3824, + "source.geo.location.lon": 4.8995, + "source.ip": "23.209.86.45", + "source.port": 80, + "tags": [ + "juniper.srx", + "forwarded" + ] + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/microsoft/_meta/config.yml b/x-pack/filebeat/module/microsoft/_meta/config.yml index 8e793bd2f9c..ee06eea9228 100644 --- a/x-pack/filebeat/module/microsoft/_meta/config.yml +++ b/x-pack/filebeat/module/microsoft/_meta/config.yml @@ -10,7 +10,20 @@ # Oauth Client Secret #var.oauth2.client.secret: "" - + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + m365_defender: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + # Oauth Token URL, should include the tenant ID #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" dhcp: diff --git a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc index 8a3facdc259..7e646e1b4fe 100644 --- a/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/microsoft/_meta/docs.asciidoc @@ -7,7 +7,8 @@ This is a module for ingesting data from the different Microsoft Products. Currently supports these filesets: -- `defender_atp` fileset: Supports Microsoft Defender ATP +- `defender_atp` fileset: Supports Microsoft Defender for Endpoint (Microsoft Defender ATP) +- `m365_defender` fileset: Supports Microsoft 365 Defender (Microsoft Threat Protection) - `dhcp` fileset: Supports Microsoft DHCP logs include::../include/what-happens.asciidoc[] @@ -20,6 +21,84 @@ include::../include/configuring-intro.asciidoc[] include::../include/config-option-intro.asciidoc[] +[float] +==== `m365_defender` fileset settings + +beta[] + +To configure access for Filebeat to Microsoft 365 Defender you will have to create a new Azure Application registration, this will again return Oauth tokens with access to the Microsoft 365 Defender API + +The procedure to create an application is found on the below link: + +https://docs.microsoft.com/en-us/microsoft-365/security/mtp/api-create-app-web?view=o365-worldwide#create-an-app[Create a new Azure Application] + +When giving the application the API permissions described in the documentation (Incident.Read.All) it will only grant access to read Incidents from 365 Defender and nothing else in the Azure Domain. + +After the application has been created, it should contain 3 values that you need to apply to the module configuration. + +These values are: + +- Client ID +- Client Secret +- Tenant ID + +Example config: + +[source,yaml] +---- +- module: microsoft + m365_defender: + enabled: true + var.oauth2.client.id: "123abc-879546asd-349587-ad64508" + var.oauth2.client.secret: "980453~-Sg99gedf" + var.oauth2.token_url: "https://login.microsoftonline.com/INSERT-TENANT-ID/oauth2/token" +---- + +*`var.oauth2.client.id`*:: + +This is the client ID related to creating a new application on Azure. + +*`var.oauth2.client.secret`*:: + +The secret related to the client ID. + +*`var.oauth2.token_url`*:: + +A predefined URL towards the Oauth2 service for Microsoft. The URL should always be the same with the exception of the Tenant ID that needs to be added to the full URL. + +[float] +==== 365 Defender ECS fields + +This is a list of 365 Defender fields that are mapped to ECS. + +[options="header"] +|====================================================================== +| 365 Defender Fields | ECS Fields | +| lastUpdateTime | @timestamp | +| severity | event.severity | +| createdTime | event.created | +| alerts.category | threat.technique.name | +| alerts.description | rule.description | +| alerts.serviceSource | event.provider | +| alerts.alertId | event.id | +| alerts.firstActivity | event.start | +| alerts.lastActivity | event.end | +| alerts.title | message | +| entities.processId | process.pid | +| entities.processCommandLine | process.command_line | +| entities.processCreationTime | process.start | +| entities.parentProcessId | process.parent.pid | +| entities.parentProcessCreationTime | process.parent.start | +| entities.sha1 | file.hash.sha1 | +| entities.sha256 | file.hash.sha256 | +| entities.url | url.full | +| entities.filePath | file.path | +| entities.fileName | file.name | +| entities.userPrincipalName | host.user.name | +| entities.domainName | host.user.domain | +| entities.aadUserId | host.user.id | +|====================================================================== + [float] ==== `defender_atp` fileset settings @@ -109,7 +188,7 @@ This module comes with a sample dashboard for Defender ATP. [role="screenshot"] image::./images/filebeat-defender-atp-overview.png[] -The best way to view Defender ATP events and alert data is in the SIEM. +The best way to view Defender ATP events and alert data is in the SIEM. [role="screenshot"] image::./images/siem-alerts-cs.jpg[] diff --git a/x-pack/filebeat/module/microsoft/fields.go b/x-pack/filebeat/module/microsoft/fields.go index 2576fcb8ac7..d76c98c273d 100644 --- a/x-pack/filebeat/module/microsoft/fields.go +++ b/x-pack/filebeat/module/microsoft/fields.go @@ -19,5 +19,5 @@ func init() { // AssetMicrosoft returns asset data. // This is the base64 encoded gzipped contents of module/microsoft. func AssetMicrosoft() string { - return "eJzsfV1zGzmS4Pv8Clw/nO0JNz3t/tgb3+xeaCX3tm5tt9ayuzcuJqICRCVJjFBAGUCRYv/6CyRQxSILRUoUQMl754duWyITmQkgkd/5LbmB9RtScaaVUTP7J0IstwLekPe9H5VgmOa15Uq+If/yJ0LI5tfkvSobAX8iZMZBlOYN/tr9+ZZIWkEP+KSEGcgSdEFt3X2MELuu4Q2Za9X0f6pBADXwhkzB0t7PS5jRRtgCl3tDZlQY2Pr1ANf2j8eUzJQmXM7BWC7nPUIuAnbk7NPVpPfFXbr6tAlq7Oe6pBY+8Qq2PtLS5X6584s9OLo/nxaA3yJUlsTyCshzLsnnT+cviF0AoQK0JStqcHXS4PLlBuMoohqMEksos6LJJVktOFsgmsZS2xiiZjtIswWVcyiJVeTZx4DVswPYc8l4CdJellHcb2C9Unr3d3dA/zLAJZcXLaJnDtGD6Czd6ZlTBzs9Tj3gDjENwm2wY9hRCF7b4dY+EEfWaO3Y5vYYWs5tIX4AQWoMn0soP6l0iP26kqC3ztsBJPwJTYfAdQ2MzzgYxKDPo517MCFXyhg+FUCWVDRgCNXwhjz7LG+kWslnL8mzD7By/7uUV1rNNRjzDK/ZnW8ME47FM85wN5LT6MHel6ifnay+UoZbvgT3g0+62fz7AEUlWNAVl3kICpu2tcidyPug7NmSckGnAkk6q63733sqVlTjT66BNZrb9RVoo6QE0f/hJ/8MuR99lisqLZTXambb7/5qF6APccYuNFD7M624WH+gI/L9yLvuIJMZgj70wkwp+zf3hqdF4bMB7XWDXUGI27IfJ1g68c5gUqqKcpkWswuEiSs9BDVen5Wlu+FRzHh9P6Qurwj14NwjgKLCPcv3RYrS0jE+5du2eWIbt6UPwo4x1Uib/qDhXqbCEqR1F3xdJ3583dcdku1Cd0TH0XOlnSZVU5GWdR1Yck8GtkhKsCulbyZcWtAzymAitxFUS9Arza2TdrqBgfEwRPt4M+FDj4aAGOkQI6sFaMDfWU1nM87IghoyBZBETQ3oZV8V72SjofcgZtcSOkDK0D7ZqIHWnWmxRd746mPr7zeBKjPfORD7VrjHOfu04MZ9jnDjzhIKV0Zr2wT+a7oiFRhD5+7f1BKmKnBizwvhwSl9p+bkApgqQccJ8bD4LlLHkrO5gSBt4UhLDDggnJn7geVBqVXSgrSoznJpLJW2RcPEdZOhvXkXBA9Zo4gd9zih6UltMD0pMWCM094W3BpCyQewv3Mr3YsYdn8SEauBWLNQjSiJhCVoMoXu3NVUGyDvwVKHGiUzrareUs/fqbl5dUXZDVjzYqgpcA3MivVLYgPelHwELyz8CZc9NCdxTwMsQRzBSaHk7v3c4uQF1BoYKi8OkxJmXEJJlBSIlnV6LaloHceqMvMi2YXZs8fvwz2/vPjOq9/+xqPxvtHe4ZYyS4Sa+/3Sg41A6jjq9v604OfcdtRUW84aQTV+P2zsZPRkDEAfdVJiJ2MAefykjG7J8rR78vr/78n+PXGr5tmQh11fNf1HgYTsbsuTwW5JjxF62VHTYFSjWaa39+Fsy3X/H4YZugsrkPYpIkebktsC/WZPET2QVq+fImILp1M9RcS4PA6xvBpTKzme7kkrgR4jPfKybQZQprShRvSamJ3Z+2DrFnDYDPSQgZLwMCtiRw8ZQD9gRYxzUQ59PyfgYt8zFGWfZ9eAzETsIxEO3pt97BRqdSP5lwY2arTu6A8/Wm8btedKMvc4UKueumU7Im6WPK847HP3fCsu1h7Id2pO3i5BWnKNwpk0mAhASa0hCKoB6TN+CyUxYB2QrS9vr2HGDZZ2EwawH2ywdJswAH2vTRl6AtP7l447mAO67sGT+/FgoUwmfbV/Ln9RxvZFpNg9kQZkyeW8/aWJHZueD+nr4e8gwnUX7u4Pi/UZe3m1/KELiY1d913mDqi36mtl7vKn3Oz96f9d9tphcC+DbNiVC96R1veWlYSSOV+C7JxkX68i4Fh0nP8irwVSPkXl7+uIaIw6NFS9LjR8ybDX/eAhbjDSPV0jl9/6pckVXqSXwZttKfm0roEwOpQgUyDA7QI0+Xwp7Xc/EaXJz0JR+/1rMqUGT1EbIJvxeaOHWUpDuo9Rd79iujEMms/4TOBfcN+eq1xutn3WcbvyV+9gUHpFdZlNqetJtB7ZfU5eXv22pe9RzM3a3VJCzNpYqMIjGtB20BbgT6rxzHP/VprPuaSi/c62tnKAD7n0rz2JEZdXv/0UYUFAf8CJh7Ogw2jI5RSvz+agDhXHY1+fBdAS9Eli17/gUuTy4iFRUo9vP1iKYI6LlT5pJ5tgRXY/G20VrcuNooUXxZku50oIYFbpr1EAO+49Qs6NO3PcEOZZ59P9thTVd2pXbSF7GP0ELb6KTZ+Kqlopg8lulZJkuh5sGiEavjRgrANoeFWLddgn92EsfwLKFsTwEsjzvxC70A15/eOPL7A8xwDIbpU9nHgSyusdOGFqJQ3kYwX7ak4F5kx3PoWmmoZqGV6FR2gXAnlOp2oJPWb4JN7hKx/Em7EaaDV6f9hXc2wemVVQ8mZXT0vBqG9immPnWOAzwu3fm9d/+e6vxov0VzUK0Bbpvw+o+buzB9/RNWjymryVjNamET6y4kzKe8n1GPQHBj8iuZWxVb5/Tf7ZkfuSfP89+WfClMZyDtwmv+hL8t+F/Z/ug9yQbaZ8E91CqUp4srauXEHBqBBTym7yasAeOaksXhtqvV3hmAiyrBWXtq2ciSKKh6MArVWm/LSNPmhqYJwKxBgxNVZpp1nLtdc63C+WVPDSH4wYUoTMVCNL98IIQOS5nAfl6GDy4vaNGEBOEQsM12FP2GhkF9ZC0fKpvHMBHWL4H0AqsJqziNURTOH+h9EW9s99K4Tds0/tRqNVs3bbJuQXtXJbM7Q5uSRKO2PMKnIDUB9g2pN48b4SpmnFwJhiycuizBV1fdtKnjlI0NTiJS8dB3t24ZJr21DhjPYt37uMuDh4xZ3Z7WsUHTM8FeGqYwF3rcGgQwWZRvUcbPexg5wwOlPS06NzwmfC7eeEzhIKGgr+TXniR6iUBXIdzjvTgA/tdD0mKN2fNhDzFQRewkqFqQXPmdnwpM15wwdq/5PQzZzMzXje8da5NyCc9fbUtVZLeEL+a0QYvXiZcfEIMXq3qjOOrs7ProLuy6h07OFVrfSuxkvwifzq0iCap+H++OyfKjTE0XSPuVK3Tflm85WNwe71HLTMJ+T1jz+RFfK9AioJFSLuKwjdINSMbPxHZAUaPFhqiQBqLFFyp1xkm4mPriZ+3UyM3NUcYdvAu9+VLpFxmNUEbCGVUPP1biBuxvVAiyXkR8IWVFNmPRPdpV4j/ug0l6SRIadHbPnMRytqUxd0+0B9ziDCntglWhSVUzKVbMMImq5GZRpK1h21kjLUWH2MQgafg2Ks0S1EY6ksqS6JVLqigv8Ry+9VuorypwxZDkezSDXTwZN0LyZtsO6QeSX4DEITrojTkSlZjijYm+0ujM3pZ9lDEJdMVbUAGz0Ao05Uigq81XxHDPbqzbR9pIN87daOHuexo7x9MkePX6WkXSTapk19aqqcl02WU/lIjH/bdqNLy3YH8g8lc3db2CMW3eqtiunTawfN/AYiKtuNPiMWbm24fGQJ2vTKKcp9eWCR/X3oYVsDTUXmpkyPKV1Cme8dDEk24Zky3YqtjtFm2nQf7MfXh6+VVtUEoTZYlG8YSKq58mp91QjLv7UcNKF1Ldrql00vm4pKOo+V5hIiMLzT2oseKY+rIdw+M0StpI+MWVrVu57BgLFbzaE4vH3WELbgzrpRJZgJed8Yi2ZSH6i7ldSO5OVSC0du0l4BNps5vJdwCk0IN7ld0PNOwww0SOYPBHWqdcmXvHSaDZ6HuCC7bgXZpx3mxYm8rbk+GYWb/fSxoFt3ErkVa0+scULP6WsOqZ0mkiTiG0246aMunJdOGnfybDJYsksnU01qCVQNFLmHQuz4n/qqoAb5pYHmZEfJnW5/ijbycUUNQSTKkXODyH2XmqkJlYIthmaQafPKZnh951UOXOsiA6p1kUN7rlOKom2gr5NDzaAr9V6RxzEhd8zH6BszeC7v9eYcKzYPybVjggWbB2KnG0JqRxBlkVa7D1esTSNyh51GrCjVWKYqeOVx6IwXzMpWs8EJoTKwYMuAHDkgsATNbc7SkT2EtauHIsBeZGefyydv8eKgd6B/pbtKF2xjSo0PwM74xvCJa7c+mDPWUyXoyvmzmSIb0LkYebkpmGhdVGUIskTxDmbzqTbht20rvW8JKk1+vQ6psdy0CQG7fjVcv92hsSpJU2ND79ORhfqgw0qWm4b03d0d7cLTCFvka110T1Ekmwo0Z/eVRVHaTlDFtoewfiVbdzO8WPL3e0DaEmSJIzkOyi01/ccjdK9pQ7tq+g9gcTvaIZa/FnzA7tAIeg9iXtLn7FX3zfBChqr/IGaCl2tBu9xiqSyhZBE6XsQTaIWaF22iyqMI9fYg3luon6Jnypbsw6b7vms1io+44q8EZ+vct2ePXLhCBEJzbdmfJtBHUzciZ950nIEfGwFk0BG9E6dKWrjNrbF2CF1K76/b9EOlZWncf/BRpaJFKNYA5sDj7EfvFBJWuWXBWOASVr1QPyoh1mo+bSz0JMQwRz9MDXLaev/5i4sOU9Nkwm4zToVna1u5j2loCO7mF3lk+vpbxLjFCjDHsLbhoNnkfOkl6Am5Buj69E/oHLCVd8h0nynd4jCA3YIJk2B8n3///V7fCqXJVKsVzgAIPw26pje7RvtJX5ZXVNvUbroOcGqPSrhTalAdeqo7pUTZqY25rpSqIQQUc73FZzKMCGuzi/Rm0fAzH94K4qPXBACTkCIKc0mkkt9qqAEtmX3ZDyYyIitvH/3YAC2vx73iPsLWhn8GlK24XQRl2ct6coELTrHaRBIlv50r9/c9LwEqKUVEccxIN+0FA18hAg5JNSM4KIWDmXTTpXwQM4r5kU1d74DxuS/na4wzYnzJqE+2KYP4DYynhInG2PZAhn8Mtgm/wo3byVATHfwbTvHF346rQCfXfvwNi1v0vi1TPqXs2SHDy2F5gVgQaoxiHP2lbjei9iRu2Dt+A28IJfVibTijgpTc3LwktcaZKC8JWPYsrihTTY+pvbznQ+/rbDStwII2pKYGu3gZbOTgexEwVVVOiqmtoP2wtAYs26vu+ffgsTS+3h5meJi8+GaqqpvhHcywbZSsuCzVKuTTMiUZ1PZll0kxyowBmbNGiDX50lDhnZ9lb5oY0t0uJNTI09X3eqZSl/aQ7lTCd1zeQBlqgdpEdGrQOxUMFPebbzrUJrzct3Fi0BUiq6jrT3byboldBFr0fr1+LLx+rYPnlVwP2/V0QWc/pDDTaIQRF2tYE7H153+/pv19Yk17xkX+O96R/DOu1l1jDWXDgLSRI4i72wxoTkUReU2zPSLXuGSrNu++j70H0L0wo34BYDfmqJYDKTzGYXX30C2oWXQ3FOfnDd+Hhi185m9bY9OVGZ63kHZahDlCumUmRjP3re7fw0pT4uS5JBxz7hrJBFDtfoSN8DaohQLC4O3UbWHn4eiDF37NsM/Tk36xmKqmvcmo/QcrlI3qe7xeS64bc2pPX18bQQTGPX6nCZBGrsS5X933ZBz3lHoLLrtrvGOf9zJfXpAPXtI83xl56ot+HW4v4nq1d0A/hi+/536+vECWhpK3TkwMvQfbETmfBuhJmPhD5GTBipu4kbo065y97LejuqFA26sLe/3Y0hvfJzw1jvXn3cLk8uKgJpvKP3dAk3WIvZblRqOdkHNfnxn6nQr/i/3aLCKotz/x3TfBHTdtbFe5qWz3GDVSgPGcUf5BWSmypJrTqRhUAfqmDFySWtARQWBAmqz9UbY2tK+q+pUnTlI5DaOtL+Run69fXV7t6tAktIz1HoWxuuwjBwreuRZyE2nxSJJLack1n0uKwmLkiNZK52xe+2wgv9whvWp1N4VdHfGvDpHeXcZTVqrIwfnw6yfCJRNNCU6chUG27usT8vztLa1qAW9wdK/TcxEsSu9J3C+CkbmTxzbRObV5WuKYcXPjVO4j8LpHKV7PjfkhPA0fubnZE3K1ms/noPONsIuz7Ld+LCDggNrpQoNZKFG60+Nt9ZFJo1uh9xN4Foax9yCVn3/0OsaLrhnH5UW8jOTO0Xmmqro4cd4V7krIvcIxrt6/Z5rptw4dJbE+dYbjZlTZsDErLailj5Q11se8k5ZKY+cBJ9db/EamxFFdrqh+nAy9YVd9J11peIgcESOtkZ87IUrJe8rafspx5daJoJPaMUp+2yqoer8U8rZm8qHWGqhJnhtsLLVNKsW580dRLh7N7HCLT9Ut4eWr8ffLvazNKTB0GH0eND72d8FhEb+67TuWefre4JBfDOfuHfOccamaVDHOXh2JmSe/U06SpnQ6DDyyPyQGnLsz49aROBPCyT1iGsbAmFkjyFu3PmGqBOOORNvsN25ZcFnCbWIGCG7scZrnA2ULLoymmG6RmILG+GZFNReYwRPx4Pn4u5wTikz81n03SpnMcA7V1DcXeiSNOKxOnnf5nDVoU4eiWy9hBiwLKsImIb7t8PRipMjQu7mG73HuhBKvfHVJXsFX5T/tfkm5NKQES7mIOBmmqrG9742QpsTJczNbjy3t8tgQj/GH1EJVi2zZPGekhBkNIaDQ+bKN4YdsTacVL0ELusZCLqvC40qeR26k+wVa3eHbMGurwL2v3lhuG2zMSKKEbWyDYcOmh17XpFGsnn+H0dSYZpBVTFWVu095jtG5h054L9m31mrJS+8/a7vIVWBGE6FKxY4PNN7fW/YzFxutkfXz8uKqwW2NSU+PI+vb1fPK+n+o6ZF+p6PJ+99qGgIw8dtV83yNcy8wodjv/PXVJbkcKFR9NLJ1rQ3VJfsxSFjY1VXDzpMa0vfxh4Xc6rhy70VEMVVl7oqvQcXdrtIRcCEOlxH1aJG+W4IPGZyg8rznAg6lwz6BtouH8Dkvu1DOiBOvSm01DsrAE7z86ZS8ju66yflMtdO9rz777jltIAqTNW6BNX0vgk/9mkKsvLXtwrQvceMEjpCoV7zcdoh01ZV0Sbmgw0AG6VzhBOsrZ6D1yKQFf4eO8fWni7sFY6UKDaB8AHZAUkg3MHw+GZGIvCqmTVmuk/tneFUkrQPqwW0MHNfofK+XKj1EzVXCLgc7JXaFaU5RkMBNP3vV91ylTcltV1m36YsWMIoNtttUbHhRsgkv7CfSZ4ml5uDyZFb5+W9vyfNQK/FbI5yuPOUCCzgwD+ztba2M++QL8u3Q0SB3ozA3Uq3kliFkgDXYzGK5DX1k0iajJ3DB7aaFnrdV7h9CadI7mFO2Jp9HzTXBp5o+RlF+WHiLxVySinI507SCvekYNdU4tTd/n4Qt5fIKlyUfVOmTozdtAXtZZxGkyAHtC1MFHCNyWUjbfeM+wIr80kg0Jd+rEgR5zuVy8ueXhCv2kkzdf8D9h0oq1oabyZ/j8UXL6mIm6GByfmodalvDP78iuCj6ulBOrtvhV2q2t1GDVVkx9T+dBjzbNggGtDvIUYSWVVq5u4PZb+9/pxrIJ58A/Oc///b+97OPb//8Z59zu6Sa8tEzuVL6JmXJ8sEL9nu7YD/CNuoEozK1EhFqdtJ2KemeA8rcc7HOYMLMlAZpOEspQHqupAwYV+m9IJH4QCqgxYry4XDiB3sHsPd5aqDu+qQuUTfNNNOlsNPSWJ268h3rtbM5xPpvabJ3tK35yOckPbbYZTMYbKDShGKTTd1LqHdxIGZ81NHUkprNEXssqdFuRBEyd8t74kL56H6C93dcOOSD/v9xuOpGZfaT/x7liJU9H31AZC+Sj3I42jjuPvyUOkHS1tbO9uzS57bLaG+z7LBP5gt0uw1O7uHIdNuymp8iHoZFXzPKheN128zlKsiMy4t+bRt24nLmoIV5pIXBeFZhm3NdOBXxCHqOSbzGdOtQfXSuqqqRu56oAXbyuMZND8XuA9zaf4O4Tt3hZo7TrB+K2zWV5b+qeNRsg5ullh8jGR6M3XDhLeRMY2rOuEqWJXoqCx6xX1Eth0GHp466kVVdqFzC+PrD+yvyq/ejbpJS44h8OWkqwfV/vCNfGtAjvVsbIQsNu5068yY39Byia/KxLTqLpnV1WjpL+JD2garUYwQc0Poox9EhqDYSHHsw3DL9gAYqqK4y7JYDm8G9QOuEBcgd0KZMNpV2C2babldboEtqd7XCh8KdgmSLiupUZSUd3HVNB+OLHxx9omyQTpUEZrFIfhYYzNIWUHWAZ3NstZQBrJr+IwPUmiafhOE7TiU/Xhh0L3jqByd0bqvAqZ7JkZYFZTgYJX35iYNtZELjvQd4Oq+XP8hbu0j+vjNZMKuL0iTtu96D7iAfF3m6A+CloMklhixAzrlMWBQ5BJ0jN1oWs8KsuGXJ5YcsZkKtDK3S5670YUu7zAc9Q9SFyYLLnOKEyxp0NV0nS3gfwK7ZTR7gSypynBVeF7VWVhXpQ1IIfflDgR7H9LBFtrsp1LwoczDbAU6f/8ZkUdHbwtpUboNtwO5EC8jwKFRcZkKay3xI18IUYiqK1GHRLdh/yQg8eWfwHuzUvRD7sFNX9fZh/5gR9k8ZYf9TRtj/IyPsv+aBbVUt6BRyiJQOenrzTBZVI1D5nq4zvJMt8Pomg15SNYLPqzqP9u20TCrmqZOQAmSeQykx8IWl943IwviExAw7aDTLY006wHmsSbM2TZ1hFimTXVl1FlPVKutMD7jNIEKsss4wywUbzZoswBvJbyWVygDLcAiXPzmuZHoUlj+p2i6AlhncaqqqCyYy+LAd4AxBEoSrp2ub3i3qIJsskOumyBDTYJpbzqjIUEBkCjoHydYJs676sCUV6z+gnObAe1lgG9AskH07mDxY+8TaLNCn83r5Ux4ftCmm3P41S6MxZoq0s+J2AGuVXFSbLNccoQLT6avcjPfxJ5u11QMMduH9/OmdIx44qn1ZgPtu8uk6yPVgz7iAHDaMKWY5NpHPUhZnbwPOoRuYgteYpFhkEXW8Xv5QGlsPmvkngm00ywJb8BnkMGMMOporKHmygtFt2FzmOSWVKhsBhqkc3A7A+TyDbFK1WVGbdOZ/D3osgzwJYA1zbqym6T0hG9gZND4NdS5W62y8NtiJXGeSrz4z3x/xDNCtBlplUCR9KVAutPMp16uF4qbwE2bTQ19TTbMc8HKkEDYF5KWfb58aLjeWyuRzjktjp41ONSywhQp+VlAOqE1yXNPr0W1NcmqwOLlhln7Y9bGdBvbBnNOyTH0HeJk6rNq2DsrwFvGqYFqpKktXIgc4g5nGqyJPcmToeJSDzfVN8vZMtUnfspTXptY8MVBBLbdN8uwzwSWka7GzgWqSTtTp4GLxbXq3llC+62kxEyr5c94Bz5Dy72ze5FLHAc0gcZwNnQHV5LkJQs2zHF05z3KBa6VTC7Bq2sxzXLOKG5ZDLFQmy4HNMQdCgsXmSsnhJpfhvgF06ow/DzV1Op5crVJbIFkqypQfAJ3cElXpNSOl+byIzON6MNyVBJ3+zaoLP5Q3Odikk6k3YP2I1yyHLEPhZpiJk1oYBLCppUFdeEdScnSpMe6XBVukqvMfgIbbmicPBNSgq7mm0g567qaAvMoCOP3T6zuRff68MwU0AWCt5gU1dcKBAX3QmqaGqoGKHPqdBoZ88F1HMwFPz2QHOW0L1x5kpcsMGKd3ZJoMvmHjfcMZ8gEMpE4E8AOPMxgnBr6kPwCxBq3JoGYwpQyfZxC8pk7tZTOa5bgHmpXJFWmjWawrbgLANt2IrT7MxiTvqrlkMnWhRHRa7EOB+iadqcm3c5v+WHmg6SN63UzP1HDXdfJurU05zZKH3miR4S1sDOii5Kmr3rOMrWgjQznYYJmxtErtDV4WXBpLZxk0gyXXNocavqxlhtZNVulGpnSzxtqiRTqKnjVWkY+NJIOlu+yRjMPyfqOCl+RcQ8ktOae6DN0MDbZ/j6PjJ2dl5NLYhFAEg0P0CfY3YEqQWKlOlw/BZT7Ova1qodYwGCx4kH8z1SRr6n3HM+Z46H1GOO9MwxxuSUV3Gy1sYrFy3uwOA8mOpOAGhzO0q4etxwZKxDR1rbQlw8ajhKwW1BJuSa1hNnYUHpCWe58hFDHGB6ujQ4FwGTq7j/SFFlzmnsjfQ9Wt1sfTEKvmYBegJ5vPm4VqBi8aIRKWoLtxRFaRmmoD5D1YihPB/V2lHQuev1Nz8+rKl72+IBdhxNdLYheRKUXYDPgjhNHHiLYkH8D+zq0EE9/n4aHOwrwZjuzubhEu7ok1QDVbTLjkUfxw5u4J+mvviE+chYHJEK8EbSTO+p03OMe1beIeb+C+0699D03523F3NHVNuMP84hFj321EkbCm6W6dV3FZ8gluLd6KMXfBKaZRjwikzeC6DzihWoqRiZfYPTfjOHDsn2vAEg1fGjB2T9Pu47OV798r36sMOJbHr+ol9q5Hqss73Xan7MPJY4Sxsa2fY4d28yZKecrZ/4fnG7rFLi9aoYBrx88GWg3pknjveYTd4zKlBohP1+6wIYNb1e1S+Mbj4Cu7UfAd5kr79vVRNhJCDTEAOO6M7p9Xpak0lJ1gvO+gw7RfWqLauzk0rNE4AW0f0jXoint141RIb5b0gzn4kguYAxGwBEGoMXwu/cZt5vXHjz62ZH5E+Y3r7znp00eZ9OwwayT/0sDumEQav3w9fI/rmHjcFJRWo+Glv5BMSQmYW0FW3C7GBAUhkcqQTmPXcFR50b1NC8dOlCfdEyXUnDMqiMNgxPRBLB4XO1xqZEzj4/GuXqxNHL1eOttK7WS1pn7gqeDUFAuV3SbwRlxnruEslc1QIycV+yN44v0AiL80Dlt808IgFiaA6smZMMoZ4lv37QKD5eSX8I0JOZPr7l8D6BZteSMtoeWEqapuLOi4GM7ixneE5TPPvtndC5yxuLUh3P69ef2X7/7qbN+L3na0HPsminY4p0XaiNldHTd0DZr8U+eTM68CGohc/Nanrv/Jf+blBuetU793P45MXj4k257tDkxx60zIh18/vXW0gwbvPEF/ackN01BTydZOqwzqmdjNBSHIoZfk0/s35FLa71+/JJcfLt7+5xvy+VLan34gz1eLNZHA7QI0YQtlwqg0pTUwi5/67qf/9d9ePItyBOwio4zb5QfK1ElF4+N4TObTd89rfu3P4mWLVPyKl08L6b5sOoD5kQ3j7vzAx/DdUUw31slvXNuGCvLu7EMU2T+UhHy+rONOxv9REiZx3jp0vxoRioQcFp64BU/xDd6zD3NqYUUfYUQ6nu4rclaWGv20/pTH0OmeXlbVx8Y5HxoLuTx/f+VfpdHwWEXNCaMfW04lr6mGt5tcXjlURrxfjodHToJIwkO39jgPW02s8NO1TisgeujSsuTuw1RsAra9Wf7xd+6EB8CZhHjBVbjhF9tHYIDKJtc6i1531yeNkg8BwyulbSeSB0K3xAAbbgC368OS15yY954eLuftY9KS9X6M8RJiduOpvLgBO7R8qTGKcadyer/RQMchTi5rKucw6UwnpuSMzxsNJZmuESbIErOG4nKmPrL1wKBodERbji46y9DvQCTU/fslXMkdABoqZaEImd3p84zSs7aUpqCFT8XPALq2Og/wWYYjMctQLSxyXIdc/U/qDEylZdF64vKp5bsWvKNjsrta35nwCBrsW7sALcGST+saXpLP7TP2Dh1g35Or1gE2eAl+HdPU2lE9J1AmRkzjFungF39JqBBRZaLefBAT3KjGxLwlaPcGcmkVMRYfcy7J58tRgcIwQTabvEoush1QVWcY++YAazCpM3od2AwlLv5FTJ2Kjv72DNj60QqFADlPPikScXbKR0YtdEQD9SoPFb0AjCQM0wlmhJKflV5RXQ7ndBNyNsdkL02ou/G3mEs3BbsCkHHVM3HXxPvGuJWloh+q88gQbBmPmREDCrkMea6YllBx68RSGLERJ3EpqDxFHP8ODso2QaTnohwQuO2y3ERSls6CnaMBu/3ypI5UAsMuBMt0/eDuFrGn2nLWCKoJ9osmLRLP396+eafmajaLT38HVtgFZN/eLWQ/uQX9bezh/dbh7dA9a+wCpA3J4qNomyZl54S7JfT4JcdR/2xAjyKsGsvUaTkdlhxH+LphDIwZwRk7jx/XHO24xBPEizgVd670mkQKEwa4nUI4beEIOzg6qYQBPlMr6d4VJ7diymH3RTJQlLapWqbrRzfyblLiu5ZizYDgUHb0BD/Mjj7MJTHcNhH5SbC4AIKIDlAX1BBaqtq9LnYBXBO1kpst84yz9FZJVY3k1eJMDsN9i/rTKhFOueeydPJHadMxgJKfuQByFhCbDNhwF2ev7Ajzd3I0Ybyj/1HSFUZZcB2yFtJyIUZjhBEp690fwAifr3cd6jVSc2I8IXSqclYPRIifwoIuuWpQu2SqqrWq+EiGIpwaubeSTgUWkc3I+X7cuFx2YicjkrsYbmmdJIrAFoZJh8scgWBk/Q6/3Lvbe2U392302G3KLBtpd8vZUmv0JZaBF+wYs/5OWhC+x3OQoDlrSUKGYKLfbmoBtwt8amOz3UhAdsK+mxirx4OfLU3HtN16NJpe76cpqBd+rYx0RU3Tzgi3vALj5LrX9jTUMBpECruQrCnEwY3AxoMP3AZ9x6N1TO/uRzta39+Npu8Kk2zI6Z1JCw7jQxQOaEOKNwLhDsLg66Xu9UHq9En3zl+0JLTpwzuXrJfqaQTIATneCZCv9zh+f3jLUo02OM2W3U0+6pNKkJR37A7y46THMSVtg8PYKfVYgrbjp05eudPYRVGBXahHiJLQLU8y8WiEj41uOPZS0iqr12lPVOejEsFf6xDZcy4zeUL+c/LjX/5Cnr+7OLt6QS64sVzOG24WUGIpfBQXoeYqe1+gfZEwzJadeTzCNuMHRzLGtMrsVdxX/+l2NYZBd2PQI59s6PN9rgvDtP+u7rfn+EOcYjFTKmNt0jeZYlSk6k63Q8hHWvLG+BWI0sTwiguqvXhyYtPdIYbvery8Cu+54eUpO430M+U/u4PQehF3+mJuLnm+Ooszue+uY1gjVBr2/L/BSYS/GZyF4LiBXllGGXdlKp0zMWAQskFWKz2nkv+xJ6ta5jsKd2X2EZzun6kRds+4jtaSZur687NbDl8L3+LL9y7aymr+BaiwC0Y1kFpDqSouabTgrieerqjlIK05mB4v6CmpfUcflVjf+hHqTAfXXZ1nTnDVVFtshrQhdb9YPWGzoyBs7iJRZ1CCphbKIllS2Z7z4YTPz+2KXfDsSqslL7vmYeFztK5F0FQHByM0/3HP2rZOG1dwNkTy8kRUdkuGXn92PUJmdHgoZk4uuY+eL3YV95EWcJ3SmXIo+H01T7hFnan3pV4l9DxCqNdRUWOlhhirtJf4DloFluJqz/BTE/epZ3HqK16WAk4n5d7jeneVc5Ht7cm9o+RcOx7jNORehdV6HYbkuo3OviS1oG7L3PusNAHJ9Loe8/JjKuQJ7Mk7ZNDpzrb8RRlL3lO24HLEpCtpJsnxzS6vP0vM9K81OPHh9CPf5MxMyLuS1uQ3/IfXj0olfd3p34ePJ1nQJTjNSQDV5EsDek2wB6GplTTQalTx4lRHb4HfOY28DD3wmIOsedsFUnryfV++cTxbkk6A6uYAfQzNUe+KKU55yusw2z3jbWvprSZGzjYMDy83RDdSRu1Y87J7eXzk2beRGqmxCxCLYGHm3whKVlyWamWIqYHxGWfuNy9jdYIhT3Z4QRx5Ht9Nzg15jh1hQbLNM4Shyxc9bpFG4jv+DuaUrclns934tovAVruFtMmza90KJzDYR177vqmFqGCtGh4y9yIOON71AYhU/29VmmI5z5B922TnV6jHuvN69TpCMVIYPWjhO0cQe5q83jFSQ4ZvcL23su4tkj7eBXRIzWkcdl3AYHtvNgmZfhsGOxRvSHG4+BnLBlKOBBytcEOSS5hxGXz1KJywq19F65Gmg4jdUYVimXDbOGB21L/UgrHz2eamPfRSGulN2fmwraVsUZ24Bf5mVWQ4GVhH/e3IMuRlymW6CWJJ74YjGYsK8z6eESHVL9vBbfFttDfl/ZGpnQOs8759B7CuqW7PlPvxyw0pqwUftFIn7nY4W9Ynv9+JPJt8Zolva6H0Ot+G/83UVP7LwY4xLSLbXdRb9Tz2NDm2/O0VQj9A26OpRAOq2n7r+6kaPQUFSKtVfYzoKFUzHTgX7nTGw5rO2oYD5QiIo6/uOO09PFdVTeW6u4947XCcvrdXlqDdM1RwOVNxpYCam9w1Qgfkx44V2WK2grxd0WdfcuUI/NwIsSb/0VDBZxxKcoF1z945GEVlBdOCKXXDHyno/jtMiV9/Yz9TMabNJ+82uwmH141FlfvIEaaH7/rHbokwZSe4o71PfkI+rWtP+sZz4Jjjd3B88zTMiqTNZHfQdjh4R4R+ZmJta3eROYWrrlMut7HznsVa6dbbjyHmj+9GtrzXKyfxcWp5UeedQ7SHFW7lg577Fk2tVCZNZBspt47bD1JTG3dNMllQkzLa3wOsQzl9YsiNFgm3uQc14a50xmjR6FTekB5MA7qg83Q25QZ08udpG3TS9Mdt0OHUZxAscGtBomqV3jhx8JOd5k7RW2jYSZVJrVH5JU5RS7glcz/hsqhevQp/Pw8ovAp/CXlNMbc/FaDj2XmBnEeMnnti+sFz9Lj2Rq0NyCnDQDRnUnE5A61H4q5Duk9CV1/xP8j6qHv2BEi2fYlnvW2IXCkMa6usVyqyxMmO31sft3fH7hNmEOv+j/4dhgla4wM/eb0AfRp/hNPZQ8bT83Mc/fiCnOP6cdRA2xM1Sxnh8znoMPwTtrIw9zTnhayh4x4jexvuFn1mep2i9+40/+NYr+T9W6PEd5tc8z/i3hp+k0mmXP77WyJhriz3G1gvqBmZAGXYqdsK9bbSLz4+XNBtdbYJUIMEl50z1jZOb+tv4gkphs9PUVGx3d+om3r4aXTQspMm3JgmudKJkDFZKp+37mExFMQQtM7qAx1sSl96vnWLk2sMTu+TTifJkOg6g4co8vNrTO3c/xj1pOdxSN5feu7BcVyEGiOKZc4XfTekGhzZUWTKwh092iRv02hyAeY3ECzqTM0NvtmMK+k/SChbfyAG43VKk8vrs39/f0Wu3DtFfpUj01c22GaqpD4G208rFccWxRBbALsxRzmR7yaE8/Ygiw2d6/p1di3CMA00jCDcSME9Wi5oPmgK+QhKrsej6woyajQgzpba5mQTPvtYLqngpT+IESR2BeHJulrvE4TIsRtYm12xnejktwmkiWEvrK1NwXEGbRbQuJU5GMLoE7hNfC7byheluV0fuFFMVVXWPnF3xNvjERxC8RL8Fdcgdi3N1C6WlaCyMOaxBt66lb0M/z1Q29ZoRbH1pcZFrfgp0qpjCHsMCGKASMWtAWQrW1ApB40zcrebCqsiIiMx2xO1be4eljDz8Pd3Zx/Cu/dqZ/nuQbFK7/r+k/ds4+amWCrR5GLAWTvHWYY5N91k7HacbyO5NeS5R8K8wG4dWNjbTtTdAU8Q6Sg1oskkzd4FXD9LbkO6wGS76GAJGjMFZo0gTEkGtXWG8rXfw5H2CqtVTunrGe8M9naEtkO0VtoS5fj7y7+exVJwo2xPfe6Unp8+wXK3wGDLxTqlvtlJtFHMv7399eryiryntxWXZTfWO76tjraTp2FuDVEcISuQMaBuH1md+hQvWUyenu2rHIvZ6Qo2H7sIvyU5u9qx5SwLUvnyInTpDVjsxVCcblMeuVdAS3H1X75uuCvMkeVQk0x9u9Ff4kzoR8puDOOq0YrvgrqVL+59SUwTSVGnhvzNWK3k/F+mgrIbwY2F8m+vws9edr/lcgYs/qsZ17CiIqrI0KnofYdQWRKjyMix1DDnxuq1s+xPKSxqahehWX+HA9nFYYAkOqVOhaYvhPb1WkzpXhfyTp/sMAdp9fpP/zcAAP//faH2Mw==" + return "eJzsvd92I7mRJ3zvp8DXF1NVPmqVu9rd87k+j78jS2q31iWVpiR1z+7xOXnAzCAJCwlkAUhS7Ot9kn20fZI9CCD/kIkkJQqgqma2L+ySRAZ+CACBiED8+Zbcw+o9KVmupJZT8ztCDDMc3pPL3q8K0LlilWFSvCd/+R0hpPszuZRFzeF3hEwZ8EK/xz/b/74lgpbQI35cwBREASqjpmo/RohZVfCezJSs+79VwIFqeE8mYGjv9wVMac1NhsO9J1PKNaz9eYC1+c8hJVOpCBMz0IaJWW8iZx4dObm9Pu59cXNe/blxqs1dVVADt6yEtY8087J/3PjDFoz2v9s54LcIFQUxrATymglyd3v6hpg5EMpBGbKkGkcnNQ5fdIiDQBVoyRdQJIXJBFnOWT5HmNpQU2sipxug8zkVMyiIkeTVJ4/q1Q70TOSsAGEuiiD2e1gtpdr82yPgX3i65OKsAXpige6Es7C7Z0Yt7fiYesQtMAXcLrBl2F4Ab8xwaZ+JMa+VsmyzawwN59aA7wBItWYzAcWtjAfs41KAWttvO0C4HRoPwE0FOZsy0Iigz6ONc3BMrqXWbMKBLCivQROq4D15dSfuhVyKV0fk1RUs7f9diGslZwq0foXH7NEnJueWxVOW42pEn6Mj+9RJ/WRl9bXUzLAF2F/cqrr7eceMCjCgSibSTMgv2togj5relTQnC8o4nXCc0kll7P9dUr6kCn9zA3mtmFldg9JSCOD9X966a8j+6k4sqTBQ3Mipab770cxB7eKMmSug5idaMr66oiPyfc+zbimTKZLedcNMaP43e4fHhXCnQTndYFMQ4rJsxwQLK95zOC5kSZmIi+wMaeJIz4HGqpOisCc8iIxVTwN1cU2oI2cvARQV9lp+KihKC8v4mHdbd8XWdkmfhS7PZS1M/I2GaxkLJQhjD/iqinz52q9bkM1Aj4Rj53OtrCZVUR6XdS1Z8kQGNiAFmKVU98dMGFBTmsOxWAcoF6CWihkr7VQNA+NhCHt/M+GqNwcPjLTAyHIOCvBvRtHplOVkTjWZAAgiJxrUoq+Kt7JR0ydMZtMS2jGVoX3SqYHG7mm+Nr3x0cfG324ClXq2sSG2jfCEfXY7Z9p+jjBt9xIK15xWpvb8V3RJStCazuzP1JBclmDFnhPCg136Qc7IGeSyABWeiKPFNkHtO53uBIIwmZ1aZMIecGLue5Z7pVYKA8KgOsuENlSYBoYO6yZDe/MxAHdZo4iOOUxoelLjTU9KNGhttbc5M5pQcgXmV2aEvRH96h8HxKqfrJ7LmhdEwAIUmUC77yqqNJBLMNRCo2SqZNkb6vUHOdNvr2l+D0a/GWoKTEFu+OqIGI+bkk/ghIXb4aIH8zjsaYAF8D04yaXYPJ9rnDyDSkGOyotFUsCUCSiIFBxhGavXkpJWYVSlnmXRDsyWNb705/zi7DunfrsTj8Z7p73DA80N4XLm1ksNFgJnx1C3d7sFP2eXo6LKsLzmVOH3/cIej+6MAem9dkpoZwwoj++U0SVZHHZN3v3fNdm+JnbUNAvyvOMrJ//McCKby/LFoFvQfYRecmgKtKxVnujufT7bUp3/5yFDd2EJwnyJ4GhdMJOh3+xLhAfCqNWXCGxudaovERgT+wFLqzE1kuPL3WkF0H2kR1q2TQGKmDbUiF4TsjN7H2zcAhbNQA8ZKAnPsyI29JAB9R1WxDgXxdD3cwAu9j1DQfY5dg2mGYl9JMDBJ7MvP4RaXQv2uYZOjVbt/P2vVutG7akUub0cqJFfumU7Im4WLK047HP3dO1drNmQH+SMnC9AGHKDwpnUGAhASaXAC6rB1KfsAQqiwVgia19eH0OPGyzNIgxoP9tgaRdhQPpJizL0BMb3L+23MQfzegJPnsaDudSJ9NX+vvxZatMXkXxzR2oQBROz5o86tG16PqSvh7+DF67HcHf7s1ifsRfXiz+2T2Jjx32TuYPZG/m1MnfxY2r2/vhfl71m+LiXQDZsygXnSOt7ywpCyYwtQLROsq9XEbAs2s9/kdYCKb5E5e/reNEYdWjIapUp+JxgrfuPh7jAOO/JCrl87oYm13iQjrw321Byu6qA5HQoQSZAgJk5KHJ3Icx3PxKpyE9cUvP9OzKhGndR80A2ZbNaDaOUhvPeR939iueNz6DpjM8I/gX77ZlM5WbbZh03I3/1DgapllQVyZS6nkTrTbvPyYvrX9b0PYqxWZtLSoheaQOlv0Q9bEttDm6nasc8+7NUbMYE5c131rWVHXxIpX9tCYy4uP7lxwALPPwBJ57PghbRkMsxbp9uow4Vx31vnznQAtRB3q5/xqHIxdlzXkkd3v5jKZLZ7630i3ay8TxL7mejjaJ10SlaeFCs6XIqOYfcSPU1CmDLvReIubF7jmmSO9a5cL81RfWD3FRbyBZGf4EWX5lPvhRVtZQag91KKchkNVg0QhR8rkEbS1CzsuIrv072w5j+BDSfE80KIK//QMxc1eTdDz+8wfQcDSDaUbZw4otQXh/BCV1JoSEdK/KvZldgzHTrU6jLic+WYaW/hDYpkNd0IhfQY4YL4h3e8l68aaOAlqPnJ/9qts0LswoKVm/qaTEY9U1Ic2wdC2xKmPlH/e4P3/1JO5H+tkIB2oD+x2A2/7D24Ae6AkXekXOR00rX3L2sWJPySXI9RP2Zjx+B2MrQKN+/I/9mp3tEvv+e/BvJpcJ0DlwmN+gR+Rdu/j/7QabJOlO+CS6hkAV8sbauWEKWU84nNL9PqwE7cEIaPDbUOLvCMhFEUUkmTJM5EwSKmyMDpWSi+LROH9QV5IxyRIxItZHKatZi5bQO+4cF5axwGyMEipCprEVhbxgOCJ6JmVeOdgYvrp+IAeUYb4H+OGx5NhpZhRWXtPhS7jkPh2j2G5ASjGJ5wOrwpnD/w2gLu+u+EcL22qem02jltFm2Y/KzXNqlGdqcTBCprDFmJLkHqHYw7Yu48b4SpimZg9bZghVZkerV9byRPDMQoKjBQ15YDvbswgVTpqbcGu1rvncRcHGwklmz2+UoWma4WfijjgnclQKNDhVkGlUzMO3HdnJCq0RBTy/OCRcJt50TKslT0FDwd+mJn6CUBsiN3++5ArxoJ6sxQWn/ax5ivoKHFz9SpivOUkY2fNHmvGYDtf+L0M2szE243/HU2TvA7/Vm1zVWi79C/nO8MDrxMmX8Bd7o7ajWOLo+Pbn2um9OhWUPKyupNjVeglfkVxcGUX8Z7o87d1WhIY6me8iVum7K191XOoPd6TlomR+Tdz/8SJbI9xKoIJTzsK/AV4OQU9L5j8gSFDiy1BAOVBsixUa6yDoTX1xN/LqZGDirKZ5tPe9+lapAxmFUE+RzIbmcrTYf4qZMDbRYQn4g+ZwqmhvHRHuoV4gfneaC1MLH9PA1n/loRm3shG73UJ/yEWHL2yVaFKVVMqVonhEUXY7KNJSsG2olzVFjdW8UwvscZJ7XqqGoDRUFVQURUpWUs99C8b1SlUH+FD7KYW8WyXoyuJKexKQOdQvmLWdT8EW4Ak7HXIpiRMHuljvTJqWfZcuEmMhlWXEwwQ0w6kSlqMAbxTbEYC/fTJkX2sg3duzgdh7byus7c3T7lVKYeaRl6vJTY8W8dFFOxQsx/rypRheX7Zbkb1KkrrawRSza0RsV04XXDor5DURUshN9Qgw8GH/4yAKU7qVTFNviwALr+9zNtgIaa5pdml4uVQFFunvQB9n4a0q3IzY6RhNp036w/74+vK2ULI+Rao1J+ToHQRWTTq0va27Yt4aBIrSqeJP90tWyKamgs1BqLiEcn3cae9GBclg1YeaVJnIp3MuYoWW16Rn0iO1oFuLw9BlN8jmz1o0sQB+Ty1obNJP6RO2ppGYkLpca2HORtgqw6dTiXsAhNCFc5GZAxzsFU1AgcrchqFWtC7ZghdVscD+EBdlNI8huN5gXnuRDxdTBZtitp3sLerA7kRm+cpPVVuhZfc2C2igiSQK+0YiLPurCObLSuJVnx4Mh23AyWceWQOVAkXsuxZb/sY8KapCfa6gPtpXs7na7qJOPS6oJgihG9g2C+y42UyMqBWsMTSDTZqVJcPvOyhRYqywB1CpLoT1XMUXROtF30akm0JV6t8jLmJAb5mPwjhlcl0+6c/YVm7vk2j6PBd0FsVENIbYjiOaBUrvPV6x1zVM/O41YUbI2uSzhrcPQGi8YlS2ngx1ChWfBmgE5skFgAYqZlKkjWybWjO6TAHsvO9tcPmmTFwe1A90t3Wa6YBlTqt0D7JR1hk9Yu3WPOWM1VbyunD6aKbAArYuRFV3CROOiKvwjSxC3N5sPtQi/rFvpfUtQKvLxxofGMt0EBGz61XD8ZoXGsiR1hQW9Dzct1ActKlF0BenbsztahafmJktXuuiJokjUJSiWP1UWBed2gCy2LRPrZ7K1J8OJJXe+B1NbgCiwJcdOuSUn/3yB6jXN066c/BPysB1tgaXPBR+w2xeC3gLMSfqUteq+GR5In/XvxYz3cs1pG1sspCGUzH3Fi3AALZezrAlUeRGh3mzEJwv1Q9RMWZN9WHTfVa1G8RFW/CVn+Sr16dkiF64RgC+uLfrdBPowVc1Txk2HGfip5kAGFdFbcSqFgYfUGmsL6EI4f11XD5UWhbb/g5cq5Q2gUAGYHZeza72TCVimlgVjD5ew7D31oxJijGKT2kBPQgxj9H3XIKut96+/sOjQFY0m7Lp2KixZ2cptTENDcDO+yIHp628B4xYzwCzDmoKDuov5UgtQx+QGoK3Tf0xngKW8faT7VKoGw4B2Q8Z3gnF1/t33e3UrpCITJZfYA8D/1uuazuwarSd9UVxTZWK76VrCsT0q/kzJQXbooc6U5EWrNqY6UrIC/6CY6i4+Eb5FWBNdpLpB/e/c85YXH70iABiEFFCYCyKk+FZBBWjJbIt+0IEWWWnr6IcaaDk97i1zL2zN889gZktm5l5ZdrKenOGAE8w2EUSKb2fS/nvLTYBKShZQHBPOm/YeA98iAAtSTgk2SmGgj9vuUu4RM4h8z6Kuj0B86tL5am2NGJcy6oJtCi9+PeMpyXmtTbMh/Q+DZcKvMG1X0udEe/+GVXzxr+Mq0MG1H3fCwha9K8uUTil7tcvwsijPEAWhWsucob/UrkbQnsQF+8Du4T2hpJqvNMspJwXT90ekUtgT5YiAyV+FFWWq6D65l0+86F2ejaIlGFCaVFRjFS+NhRxcLYJclqWVYnLt0X6YWgMm36ruufvgpTS+3homuJic+M5lWdXDM5hg2ShZMlHIpY+nzaXIoTJHbSTFKDMG05zWnK/I55py5/wset3EcN7NQFyOXF19r2csdWnL1K1K+IGJeyh8LlATiE41eqe8gWL/8k0L7ZgV2xaOD6pCJBV1/c5Ozi2xCaCB9/HmpXB9rLznldwMy/W0j86uSWGi1ggjLlY/JqJ1+3+7pv19ZE17ynj6M95O+SccrT3GCoo6B9K8HEHY3aZBMcqzwG2a7BK5wSEbtXnzfuxdgPaGGfULQH6v9yo5EMNj7Ee3F92c6nl7QrF/3vB+qPO5i/xtcmzaNMPThtJGiTA7kXaYY61y+63252GmKbHyXBCGMXe1yDlQZX+FhfA6aD6B0Hs7VZPYufv1wQm/eljn6Yu+sXJZTnqdUfsXlk8bVU+4vRZM1frQnr6+NoIAxj1+h3kgDRyJUze6q8k47il1Flxy13jLPudlvjgjV07SvN5oeeqSfi22N2G92jmgX8KX33M/X5whS33KWysmht6D9Rc5FwbopnDsNpGVBUumw0bqQq9S1rJff9X1CdpOXdjqxxbO+D7grrGsP20HJhdnOzXZWP65HZqsBfZOFJ1Ge0xOXX6mr3fK3R+2a7MIUK1/4rtvvDtuUps2c1Oa9jKqBQftOCPdhbKUZEEVoxM+yAJ0RRmYIBWnI4JAg9BJ66OsLWhfVXUjH1tJZTWMJr+Q2XW+eXtxvalDE18y1nkUxvKy92wo+OhcyO6lxYEkF8KQGzYTFIXFyBatpEpZvPbVQH7ZTXrd6G4SqzriPy2Q3lnGXVbIwMa5+nhLmMh5XYAVZ76Rrf36MXl9/kDLisN7bN1r9Vwki9L7OOwXwZe5g79tonOqu1rCyJi+tyr3HriekIrXc2Ne+avhE9P3W55cjWKzGah0LezCLPul/xbgMaB2Oleg55IXdvc4W32k0+ja0/sBPAvDt3cvlV9/cjrGm7YYx8VZOI3k0a/zuSyr7MBxV7gqPvYK27g6/56uJ99aOFJgfuoU283Ios7HrDSvlr5Q1FgfeSstpcLKA1auN/hGusRRVSypepkIvWFVfStdqb+I7CRGSiO/tkKUkkuaN/WUw8qtFUEHtWOk+LZRUNV2KeRszehNrRVQHT02WBtq6liKc+uPooy/mNlhB5/IB8KKt+P3l71Z60MgtIjuBoWP3VmwKMJHt7nHEnffG2zys2HfvX2uMyZkHeuNs5dHomfRz5SVpDGdDgOP7B8jE05dmXFtS5xwbuUe0XWeg9bTmpNzOz7JZQHabomm2G/YsmCigIfIDOBMm/00z2fKFhwYTTHVgJiAwvfNkirGMYIn4MFz7+9iRigy8Vv73eDMRIJ9KCeuuNALacR+dPK6jeesQOnKJ906CTNgmVcRuoD4psLTm5EkQ+fmGt7HqQNKnPLVBnl5X5X7tP0jZUKTAgxlPOBkmMja9L43MjXJDx6b2XhsaRvHhjjGL1IDZcWTRfOckAKm1D8B+cqXzRu+j9a0WvECFKcrTOQy0l+u5HXgRNo/oNXtvw3TJgvc+eq1YabGwowkOLHONhgWbHrucY36itXz7+Q0NtIEsiqXZWnPU5ptdOqoE9YL9q2UXLDC+c+aKnIl6NFAqELm+z80Pt1b9hPjndaY9+PywqrBQ4VBTy8j65vR08r6f8rJnn6nvaf33+TEP8CET1fF0hXOPcOAYrfyN9cX5GKgUPVhJKta67NLtiOImNjVZsPOohrST/GH+djqsHLvREQ2kUXqjK9Bxt2m0uGxEItlRD2ax6+W4J4MDpB53nMB+9RhF0DbvoewGSvap5wRJ14Z22ocpIFHuPnjKXntvKs65TXVdPe+vnPVc5qHKAzWeIC87nsRXOjXBELprU0Vpm2BGwdwhAS94sW6Q6TNrqQLyjgdPmSQ1hVOML9yCkqNdFpwZ2gfX3+8dzdvrJS+AJR7gB1MyYcbaDY7HpGIrMwmdVGsovtnWJlFzQPq0a017FfofKuXKj5FxWTEKgcbKXaZrg+RkMB0P3rV1VyldcFMm1nX1UXziEKN7bqMDSdKuueF7ZN0UWKxObg4mFV++ss5ee1zJX6pudWVJ4xjAgfGgZ0/VFLbT74h3w4dDWLzFeZeyKVYM4Q05DUWs1isUx/ptJnTA7jgNsNCT5ss9yufmvQBZjRfkbtRc42ziaIvkZTvB15jMROkpExMFS1hazhGRRV27U1fJ2FNubzGYcmVLFxwdFcWsBd1FgBFdmhfGCpgGZHKQlqvG3cFS/JzLdCUvJQFcPKaicXx748Ik/kRmdj/Afs/VFC+0kwf/z78vmjyKptyOuicH1uHWtfwT68JDoq+LpSTq6b5lZxuLdRgZFKk7rcTj7Mpg6BB2Y0cBLQo48rdDWS/XP5KFZBbFwD8+9//cvnryafz3//exdwuqKJsdE8upbqPmbK884D92gzYf2EbdYJREVuJ8Dk7cauUtNcBze11sUpgwkylAqFZHlOA9FxJCRCX8b0ggfeBWESzJWXD5sTP9g5g7fPYRO3xiZ2irutJokNhJoU2KnbmO+ZrJ3OI9e/SaPdok/ORzkm6b7JL1xhsoNL4ZJMu78Xnu1gSUzbqaGqmmswRu+9Ug9WIAtPcTO8JC+W96wk+3XFhwXv9/9Nw1E5ldp3/XmSLFT0fvQeyFeSLbI7mHXcbPikPELS1trI9u/S1aSPamyg7rJP5Bt1ug527+2W6KVnNDvEehklfU8q45XVTzOXay4yLs35uG1bisuaggVmghMF4VGETc51ZFXGP+ewTeI3h1j776FSWZS02PVEDdGK/wk3PRXcFD+ZvENapW2x6P836udhuqCj+KsOvZh02Qw3bRzI8G91w4DVwutYVy5mMFiV6KAse0S+pEsNHhy8duhZllclUwvjm6vKafHR+1C4oNQzk80FDCW7+/QP5XIMaqd1ac5Ep2KzUmTa4oecQXZFPTdJZMKyr1dLziBdpn6iM3UbAEq32chztomoCj2PPplvEb9BAOVVlgtWyZBO4F2gVMQG5JVoX0brSrtGMW+1qjXRBzaZW+Fy6ExD5vKQqVlpJS3dV0UH74me/PtF8EE4VhWY2j74XcpjGTaBqCU9nWGopAVk5+WcCqhWN3gnDVZyKvr3w0T1jsS8cX7mtBKt6RgctMppjY5T46SeWthYRjfce4cmsWvxRPJh59Ps9F1luVFboqHXXe9Qt5f1enh5BeMFpdIkhMhAzJiImRQ5Jp4iNFtk000tm8ujyQ2RTLpealvFjV/q0hVmko57g1SUXGRMpxQkTFahysooW8D6gXeX3aYgvKE+xV1iVVUoamcV/kkLqiz9m6HGMT5snO5tczrIiBbMt4fjxb7nISvqQGRPLbbBO2O5oDgkuhZKJRKCZSAe64jrjE57FfhZdo/2HhMSjVwbv0Y5dC7FPO3ZWb5/2Dwlp/5iQ9r8mpP3/JqT9pzS0jaw4nUAKkdJSj2+eiaysOSrfk1WCe7IhXt0n0EvKmrNZWaXRvq2WSfksdhCSp8xSKCUaPufxfSMi0y4gMcEKapWnsSYt4TTWpF7pukrQizQXbVp1ElPVSGNND3hIIEKMNNYwS0UbzZokxGvBHgQVUkOeYBMufrRcSXQpLH6UlZkDLRK41WRZZTlP4MO2hBM8kiBdNVmZ+G5RS1knoVzVWYI3jVwxw3LKEyQQ6YzOQOSriFFXfdqC8tVvUExS4F5kWAY0CWVXDiYNahdYm4T6ZFYtfkzjg9bZhJk/JSk0lussbq+4DcJKRhfVOskxR6qQq/hZbtr5+KP12uoRBjN3fv74zhFHHNW+JMRdNfl4FeR6tKeMQwobRmfTFIvIpjGTs9cJp9ANdMYqDFLMkog6Vi3+WGhTDYr5R6KtVZ6ENmdTSGHGaHQ0l1CwaAmj67SZSLNLSlnUHHQuU3DbE2ezBLJJVnpJTdSe/z3qoQjyKIQVzJg2isb3hHS0E2h8CqpUrFbJeK2xErlKJF9dZL7b4gmoGwW0TKBIulSgVLDTKdfLuWQ6cx1m41NfUUWTbPBiJBE2BuWF628fmy7ThorofY4LbSa1itUssKEKrldQCqp1dKzx9egmJzk2WezcMI3f7HrfSgPbaM5oUcQ+A6yI/azalA5KcBexMsuVlGWSqkSWcAIzjZVZmuBIX/EoBZur++jlmSodv2Qpq3SlWGSinBpm6ujRZ5wJiFdip6Oqo3bUaeli8m18txaXruppNuUy+nXeEk8Q8m9t3uhSxxJNIHGsDZ0AavTYBC5nSbaumCU5wJVUsQVYOalnKY5ZyXSeQiyUOsmGTdEHQoDB4krR6UaX4a4AdOyIP0c1djieWC5jWyBJMsqkawAd3RKV8TUjqdgsC/TjejbdpQAV/86qMteUNzrZqJ2pO7KuxWuSTZYgcdP3xIktDDzZ2NKgypwjKTpcqrX9Y5bPY+X5D0jDQ8WiPwRUoMqZosIMau7GoLxMQjj+1esqkd3dbXQBjUBYyVlGdRWxYUCftKKxqSqgPIV+pyBHPriqo4mIx2eypRy3hGuPslRFAsTxHZk6gW9YO99wgngADbEDAVzD4wTGiYbP8TdAqEBrNKoJTCnNZgkEr65ie9m0ylOcA5UX0RVprfJQVdwIhE28Flt9mrWOXlVzkYvYiRLBbrHPJeqKdMaevpmZ+NvKEY3/otf29IxNd1VFr9ZaF5Mkcei14gnuwlqDygoWO+s9SduK5mUoBRtMrg0tY3uDFxkT2tBpAs1gwZRJoYYvKpGgdJORqhYx3ayhsmiBiqIntZHkUy3IYOg2eiRhs7xfKGcFOVVQMENOqSp8NUON5d/DcFznrIRcGusQimSwiT7B+ga55CSUqtPGQzCRjnPnZcXlCgaNBXfybyrraEW9H7nHLA+dzwj7nSmYwQMp6Wahhe4tVszqzWYgyUFyprE5QzO6X3osoER0XVVSGTIsPErIck4NYYZUCqZjW+EZYblPaUIRYry3OloIhAlf2X2kLjRnInVH/h5UO1ofpyZGzsDMQR13n9dzWQ9uNEIELEC17YiMJBVVGsglGIodwd1ZpS0LXn+QM/322qW9viFnvsXXETHzQJciLAb8CXzrY4QtyBWYX5kRoMPrPNzUSZg3xZbd7SnCwd1kNVCVz4+ZYEF82HP3APW1N8Qn9sLAYIi3nNYCe/3Oauzj2hRxDxdw36jXvmVO6ctxt3Nqi3D7/sUjxr5diCxiTtPjKq/isOQWHgyeijF3wSG6UY8IpK5x3RV2qBZ8pOMlVs9N2A4c6+dqMETB5xq02VK0e/9o5afXyncqA7blcaM6ib3pkWrjTtfdKdswOUT4Nrb2e6zQrt8HZx6z9//u/oZ2sIuzRijg2OG9gVZDvCDeJ25he7lMqAbiwrVbNGRwqtpV8t94GbyibQXfIpfKla8PspEQqokGwHZndHu/KkWFpvkB2vsOKky7oQWqvd2myWuFHdC2ga5AlcypG4cC3Q3pGnOwBeMwA8JhAZxQrdlMuIXr+vWHtz6WZH5B+Y3jb9npkxfp9GyR1YJ9rmGzTSINH74e3v0qJu7XBaXRaFjhDmQuhQCMrSBLZuZjgoKQQGZIq7Er2Cu96MmmhWUnypP2iuJyxnLKiUUwYvogipdFh0ONtGl8Od5V85UOw+uFsy3lRlRr7AueckZ1NpfJbQJnxLXmGvZS6ZoaWanYb8ETrgdA3KGxaPFO841Ycg5UHZ9wLa0hvnbezvCxnPzsv3FMTsSq/WlA3aAtr4UhtDjOZVnVBlRYDCdx49uJpTPPvtlcC+yxuLYgzPyjfveH7/5kbd+z3nI0HPsmCNvv0yzui9ljHTd0BYr8a+uT0289DAQXPvWx83/S73nRYV7b9VvXY8/g5V2y7dVmwxQ7zjG5+nh7bucOCpzzBP2lBdO5goqKfGW1Sq+e8c1YEIIcOiK3l+/JhTDfvzsiF1dn5//xntxdCPPjH8nr5XxFBDAzB0XyudS+VZpUCnKDn/rux////3nzKsgRMPOEMm6THyhTj0sabsejE+++Jx7zG7cXLxpQ4SNefFmg+7JpB/I9C8Y9+oIP4d1QTDvr5BemTE05+XByFQT7mxSQzpe13874H1LAcZi3Fu5XI0JxIruFJy7Bl3gHb1mHGTWwpC/QIh139zU5KQqFflq3y0Nw2qs3L6t93zmf+xZycXp57W6l0eexkuoDvn6sOZWcpurvbnJxbaGMeL8sD/fsBBGFh3bscR42mljmumsdVkD04NKiYPbDlHcPtr1e/uF77oAbwJqEeMClP+Fn61tgAKWLtU6i1z32SqPkyiO8lsq0InkgdAt8YMMFYGa1W/LqA/PezYeJWXOZNNO6HGO8gJDdeCgvrkeHli/VWubMqpzObzTQcYiVy4qKGRy3plMuxZTNagUFmayQJogCo4bCcqbas/TAIGl0RFsODjpNUO+AR9T9+ylc0R0ACkppIPOR3fHjjOKzthA6o5kLxU9AujIqDfFpgi0xTZAtzFMch1T1T6oETKVF1nji0qnlmxa8ncfx5mh9Z8ILaLDnZg5KgCG3qwqOyF1zjX1AB9j35LpxgA1ugo9jmlrTqucAysSIadyA9n7xI0I5DyoTVfdBDHCjCgPzFqDsHciEkUQbvMyZIHcXowIlxwDZZPIqusi2RGWVoO2bJaxAx47otWQTpLi4GzF2KDr62xOgda0VMg5iFr1TJGK2ykdCLXREA3UqD+W9BxhBcgwnmBJKfpJqSVUx7NNNyMkMg70UofbEP2As3QTMEkCEVc/IVROf+sYtDeX9pzoHhmDJeIyMGMyQCR/nimEJJTNWLPkWG+EpLjgVh3jHf4SDsgkQ6bkoBxNcd1l2LykLa8HO0IBdv3liv1RCjlUIFvHqwT3uxZ4qw/KaU0WwXjRpQLw+f3j/Qc7kdBru/g55ZuaQfHnXwN7aAd1p7OE+t7gt3JPazEEYHyw+ClvXMSsnPC6gxw05Dv1OgxoFLGuTy8Ny2g85DvimznPQegQzVh7frzjafoEniItYFXcm1YoEEhMG2A4hnNYwwgZGK5XwgU9XUth7xcqtkHLYfpEMFKX1WS3i1aMbuTcpcVVLMWeAMyja+Xg/zIY+zATRzNQB+UkwuQC8iPZU51QTWsjK3i5mDkwRuRTdkjnGGfoghSxH4mqxJ4dmrkT9YZUIq9wzUVj5I5VuGUDJT4wDOfHAjgdseIyzV7QTc2dyNGC8nf+LhCuMsuDGRy3E5UJojgFGxMx3fwYjXLzejc/XiM2J8YDQiUyZPRCY/ATmdMFkjdplLstKyZKNRCjCocGdCzrhmEQ2JafbsTGxaMVOQpCbCNe0ThIEsIYwanOZPQAGxm/xpV7d3i3bnbfRbdelWdbCbKazxdboC0wDz/J9zPpHaUF4H89AgGJ5MyVkCAb6bYYWMDPHqzbU2414sMf5d8faqPHHz2ZO+5TderE5vds+J69euLESzitomrZGuGElaCvXnbanoILRRyS/CtGKQuxcCCw8+MxlUI/cWvvU7n6xrfX94+b0XaajNTl99NS8w3jXDAdzwxl3AuERwuDrnd27nbNTB107d9CizE3tXrlotVQPI0B2yPFWgHy92/H73UsWq7XBYZbscfJRHVSCxDxjj5AfB92OMec22IytUo8paBt+6uiZO7WZZyWYuXyBVxK65kkmDob/2OiCYy0lJZN6nba86nyS3PtrLZAt+zKRJ+Q/jn/4wx/I6w9nJ9dvyBnTholZzfQcCkyFD2LhciaT1wXa9hKG0bJTh8MvM35wJGJMycRexW35n3ZVQwjaE4Me+WhNn59yXHIM+2/zfnuOP8QUejOlIlQmvYsUozxWdbqNiXyiBau1G4FIRTQrGafKiScrNu0ZyvFeD6dX4TnXrDhkpZF+pPyd3QiNF3GjLmZ3yNPlWZyIbWcdnzV8pmHP/+udRPiXwV7wjhvopWUUYVemVCkDAwZPNshqqWZUsN+2RFWLdFvhsczeg9P9PTXC7ilTwVzSRFV/frLD4W3hSny52kVrUc0/A+VmnlMFpFJQyJIJGky464mna2oYCKN3hsdzesjZfqAvOllX+hGqRBvXHp1XVnBVVBkshtRNdbtYPWCxIy9sHiNRp1CAogaKLFpQ2Zb9YYXPT82I7ePZtZILVrTFw/znaFVxr6kONoYv/mOvtXWdNqzgdJNkxYFm2Q7pa/2Z1cg0g81DMXJywdzr+XxTcR8pAdcqnTGbgj9V84QH1Jl6X+plQs8CE3U6KmqsVBNtpHIS31IrwVAc7RV+6th+6lV49iUrCg6Hk3KXON5j5VxgeXtyby8517THOMx0r/1ovQpDYtW8zh6RilO7ZPZ+loqAyNWqGvPyYyjkAezJR0TQqda2/FlqQy5pPmdixKQraCLJ8c0mr+8ERvpXCqz4sPqRK3Kmj8mHglbkF/zB6UeFFC7v9B/Dy5PM6QKs5sSBKvK5BrUiWINQV1JoaDSqcHKqnW+G3zmMvPQ18HJLWbGmCqRw03d1+cZxNlM6ANRuA33yxVEfixS7PKV1mG3u8aa09FoRI2sb+ouXaaJqIYJ2rD5qbx738uzKSI3k2HmKmbcw0y8EJUsmCrnURFeQsynL7V+OQnmCPk52eEDs9BzeLuaGvMaKsCDy7hrCp8s3PW6RWuA9/gFmNF+RO71e+LZ9gS03E2mjR9faEQ5gsI/c9n1TC6FgrhpuMnsjDjje1gEIZP+vZZpiOs+QfevTTq9Qj1Xndep1YMY4w+BG89/ZY7KHiesdm6qP8PWu90bWnePUx6uADmdzGIdd+2CwvjZdQKZbhsEKhQtS7E5+xrSBmC0BRzPccMoFTJnwvnoUTljVr6TVSNFBRLdXolgibJ0DZkP9iy0YW59t6rn7WkojtSlbH7YxNJ+XBy6B342KDCcD66i/HEmavEyYiNdBLOrZsFPGpMK0l2dASPXTdnBZXBntLr0/0LVzgDrt3bcDdUVVs6fsr4+6qSznbFBKndjTYW1ZF/z+qOmZ6D1LXFkLqVbpFvzPuqLiLzsrxjRA1quoN+p56GqybPnzW6S+Y24vphINZtXUW98+q9FdkIEwSlb7iI5C1pOBc+FRe9yPaa1t2JGOgBhddsdhz+GpLCsqVu15xGOH7fSdvbIAZa+hjImpDCsFVN+nzhHaIT82rMgG2RLSVkWffk4VI/BTzfmK/HtNOZsyKMgZ5j0752AQyhImWS7lPXuhR/dfYULc+J39TPmYNh+92mz3HF7VBlXuPVuY7j7rn9ohfJcd7452Pvljcruq3NQ7z4FljlvB8cVTMM2iFpPdgG0xOEeEeqVDZWs3wRzCVdcql+vonGexkqrx9uMT86cPI0veq5UTeTs1vKjS9iHawgo78k7PfQNTSZlIE1kHZcex60EqasKuyVxkVMd87e8RVj6dPjLlWvGIy9yjGnFVWmM0q1Usb0iPpgaV0Vk8m7IjHf16WicdNfxxnbTf9QkECzwYEKhaxTdOLP1ou7lV9OYKNkJlYmtUbohD5BKuydxbHBbVq7f+36cewlv/Dx/XFHL7Uw4qHJ3np/OCr+duMv3Hc/S49lqtDaZT+IZo1qRiYgpKjby7Dud9kHn1Ff+drA+6Zw8AsqlLPO0tQ+BI4bO2THqkAkMcbPudu3d7u+1uMYJY9X/1dxgGaI03/GTVHNRh/BFWZ/cRT69PsfXjG3KK44ehgTIHKpYywudTUL75J6xFYW4pzgtJn457jOwtuB30le5Vit660uy3fb2STy+NEl5tcsN+C3tr2H0imXLx93MiYCYNcwtYzake6QCl80OXFeotpRt8vLmgXepkHaAGAS4be6wpnN7k34QDUjSbHSKjYr2+Udv18Ha00bKVJkzrOrrSiZQxWCqdt+55byiIEJRK6gMdLEpfep7bwckNPk5vk04HiZBoK4P7V+TXNxjauf0y6knP/UA+XXpuwTguQrXm2SLljb75pOod2UEwRWa3Hq2jl2nUqQize/AWdaLiBt907Ur6FxLK1j8Sje91UpGLm5O/X16Ta3tPkY9ipPtKhzZRJvU+aG+XMowWxVA+h/xe7+VEfpwQTluDLNR0rq3X2ZYIwzBQ34Kwk4JbtFxQbFAU8gWUXIejrQoyajQgZkNNfbAOn32UC8pZ4TZiAMSmIDxYVettghA5dg8rvSm2I+38JoA0Mu25MZXOGPagTUIalzIFQ3L6BZwmNhNN5otUzKx2nKhclmXSOnGPxO1weIdQOAV/yRTwTUsztotlyanItH6phrd2ZCfDf/WzbXK0gmhdqnFWSXaIsOoQYIeAIAIEFbYGkK35nAoxKJyRutyUHxWBjLzZHqhsc3ux+J6Hv344ufL33tuN4dsLxUi16fuPXrON6ftsIXmdigEnTR9n4fvctJ2xm3a+tWBGk9cOhH6D1TowsbfpqLtBniDo4Gx4nUiaffBY7wQzPlzgeD3pYAEKIwWmNSe5FDlUxhrKN24NR8orLJcppa9jvDXYmxbaFmgllSHS8vfnv56EQnCDbI+976SaHT7AcjPBYM3FOqGu2EmwUMzfzj9eX1yTS/pQMlG0bb3Dy2rndvAwzLUmiiPT8tMYzG7btFr1KZyyGD0822U5ZtPDJWy+dBJ+M+Xkaseas8xL5YszX6XXo9iKkB9uUV64VkAz4/I/fd5wm5gjiqEmGft0o7/EmtAvFN3o21WjFd8+6pYuufeI6DoQok41+bM2SorZXyac5vecaQPFn9/63x21f2ViCnn4T1OmYEl5UJGhE977DqGiIFqSkW2pYMa0UStr2R9SWFTUzH2x/hYD2cQwAIlOqUPBdInQLl8rl6pXhbzVJ1vkIEwvJqUrFJArqeXUHJff//hDVsAURLHmnQ9veWuqUQ3vyQRM3wlQwJTW3GR4IN6TKeVrechr81kP37+URc0BTzwTM1/O4rKBR848MnJye92/qredPCZy1AIvNtn6aJb/ZbB779wV0mqXym4RBZUC9NPazdIMO9IVCFyMykUCcB8FX5FKVjVvAuCxsxHWm3CjWSExActaXFGUFjNA48z34JT4Q/P5IysLKqpMIzhaQj6XwZJCo2akza//+KA43LPmedWLornYyu0CDKjS677xANy4KHEvKdYG2WTUMbl16eYVKLNqjixV8J5cSXOyoIxbcXhETipzRC4pX1IFR+QG8loxs7oGpaU157tf3brDcUTuxJIKA8WNnBr3rY928cZWolfO/SZwHT2DHbdj18xFf9CRcHpvHd9uhvk85xwsBai1MKuRlFA62/StP2PUE6UoZvzktTayROqBJre9s2WFHTzQsuJgxciU0xmh7mRaOs0Htf9q4zPM51TR3IBi2owdvODDQbQd31/sWq9zmlxLrdmEw9pef3Un7oVcildH5NUVLO3/XYhrJWcKtH6FKsCrT6AlX0AxUtQGK5Y692mKw5yvHd/HzuUne8NdS80MW4D9xa2qu5/HMkhAGX38wteT07QRC3pz1jfqVuTKL9QtG5HqxVC87BIhrARXEcEhWlJNmmG2Yom90duQWb8HNHlNNbmC5RE5ye2iHll9q9mqb7ZjgwWoYa+3Z0rahmqzVRW4636LoPN4cgW4yROvGxYUJDjYjsXjVJu7yo6aei/ZkUjthtpxKntXVsyjeevqx8jS62b97iqsIEax2QzavubN0dyKtdagbgYulGdivOmiEZqdZcfZhsMqQJjHfBOKWH72dnelaFy5UcEMwwQ1N6bPonYxytv3fqLbAyGu3SBNxMIjNEB/kRyRtWvkiPQvEStuRM3543dtZO3uovOhND7VteFe6Q19YCvS/4oaebOE2/iSXA/u1E6PhrApEdJV4vIaACLYunw0N1LFteZufRmEhb3TUPk9stCoWPmiWxsq9COFo5MJP9GS8VVswJiYMkXS+8IrmVFwC/kcNbaICsxtU+vknpiWPNrxlLt3yLatw+XF7adzcnJ7+y+nf//f//N/kamiJSylut+KHOOSGfh/rAbBwM9Cf+6JO3GPNfsmWGOx0WjRvzmBxilx5Ir4uXvKyCNrP7FFc/WvHXXWecyc8NWgjshFdUTuFD/C/o1H5Nq5Ney5Z/yv8sH94xK0pjNwP5zyWhv71U/et/Y4bvnianG34omv2NbPK+rf2o+DVlLGJ/LhjOmK0+iHBUjhCAdR+rGfBDSUgfFskOv1y54BMnf7468R1f4TwplGH1wJhhbUUCshsRpfewatCO9tz8dh1Zvu3giMdDRbNajhoStCVrqT9Dh0CnJWDV7ong2wJRsDYzh36rks9IXmIuArgLMFqNXJsNDQ88+1p934gWKw06tUf7N6QGzzq9HXCFJv3oHbvhvdNeKO1jPRx5ejG/jRDR55Bs170c9sERH8p6aKqEWGZbGIHYAw0T1Qdd2zmV670dnU/b4Rcs03jsm5852+Jz///fy/Zx8+np58yC5PTn++uDp/2myH2ZyxJts/Dv1XxHaJFqNvF2NgseJxXK0LSSKB7ubrsL51kfYVZcMd9nj4BVgzPvaR7kI6PHA3zDNwsiqJetEAurhulYxHY3NzCgOacmoMCHgipEaAI+FNbrHws83v/k8AAAD//7KHlgw=" } diff --git a/x-pack/filebeat/module/microsoft/m365_defender/_meta/fields.yml b/x-pack/filebeat/module/microsoft/m365_defender/_meta/fields.yml new file mode 100644 index 00000000000..3656cd7c1de --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/_meta/fields.yml @@ -0,0 +1,176 @@ +- name: microsoft.m365_defender + type: group + release: beta + default_field: false + description: > + Module for ingesting Microsoft Defender ATP. + fields: + - name: incidentId + type: keyword + description: > + Unique identifier to represent the incident. + - name: redirectIncidentId + type: keyword + description: > + Only populated in case an incident is being grouped together with another incident, as part of the incident processing logic. + - name: incidentName + type: keyword + description: > + Name of the Incident. + - name: determination + type: keyword + description: > + Specifies the determination of the incident. The property values are: NotAvailable, Apt, Malware, SecurityPersonnel, SecurityTesting, UnwantedSoftware, Other. + - name: investigationState + type: keyword + description: > + The current state of the Investigation. + - name: assignedTo + type: keyword + description: > + Owner of the alert. + - name: tags + type: keyword + description: > + Array of custom tags associated with an incident, for example to flag a group of incidents with a common characteristic. + - name: status + type: keyword + description: > + Specifies the current status of the alert. Possible values are: 'Unknown', 'New', 'InProgress' and 'Resolved'. + - name: classification + type: keyword + description: > + Specification of the alert. Possible values are: 'Unknown', 'FalsePositive', 'TruePositive'. + - name: alerts.incidentId + type: keyword + description: > + Unique identifier to represent the incident this alert is associated with. + - name: alerts.resolvedTime + type: date + description: > + Time when alert was resolved. + - name: alerts.status + type: keyword + description: > + Categorize alerts (as New, Active, or Resolved). + - name: alerts.severity + type: keyword + description: > + The severity of the related alert. + - name: alerts.creationTime + type: date + description: > + Time when alert was first created. + - name: alerts.lastUpdatedTime + type: date + description: > + Time when alert was last updated. + - name: alerts.investigationId + type: keyword + description: > + The automated investigation id triggered by this alert. + - name: alerts.userSid + type: keyword + description: > + The SID of the related user + - name: alerts.detectionSource + type: keyword + description: > + The service that initially detected the threat. + - name: alerts.classification + type: keyword + description: > + The specification for the incident. The property values are: Unknown, FalsePositive, TruePositive or null. + - name: alerts.investigationState + type: keyword + description: > + Information on the investigation's current status. + - name: alerts.determination + type: keyword + description: > + Specifies the determination of the incident. The property values are: NotAvailable, Apt, Malware, SecurityPersonnel, SecurityTesting, UnwantedSoftware, Other or null + - name: alerts.assignedTo + type: keyword + description: > + Owner of the incident, or null if no owner is assigned. + - name: alerts.actorName + type: keyword + description: > + The activity group, if any, the associated with this alert. + - name: alerts.threatFamilyName + type: keyword + description: > + Threat family associated with this alert. + - name: alerts.mitreTechniques + type: keyword + description: > + The attack techniques, as aligned with the MITRE ATT&CK™ framework. + - name: alerts.entities.entityType + type: keyword + description: > + Entities that have been identified to be part of, or related to, a given alert. The properties values are: User, Ip, Url, File, Process, MailBox, MailMessage, MailCluster, Registry. + - name: alerts.entities.accountName + type: keyword + description: > + Account name of the related user. + - name: alerts.entities.mailboxDisplayName + type: keyword + description: > + The display name of the related mailbox. + - name: alerts.entities.mailboxAddress + type: keyword + description: > + The mail address of the related mailbox. + - name: alerts.entities.clusterBy + type: keyword + description: > + A list of metadata if the entityType is MailCluster. + - name: alerts.entities.sender + type: keyword + description: > + The sender for the related email message. + - name: alerts.entities.recipient + type: keyword + description: > + The recipient for the related email message. + - name: alerts.entities.subject + type: keyword + description: > + The subject for the related email message. + - name: alerts.entities.deliveryAction + type: keyword + description: > + The delivery status for the related email message. + - name: alerts.entities.securityGroupId + type: keyword + description: > + The Security Group ID for the user related to the email message. + - name: alerts.entities.securityGroupName + type: keyword + description: > + The Security Group Name for the user related to the email message. + - name: alerts.entities.registryHive + type: keyword + description: > + Reference to which Hive in registry the event is related to, if eventType is registry. Example: HKEY_LOCAL_MACHINE. + - name: alerts.entities.registryKey + type: keyword + description: > + Reference to the related registry key to the event. + - name: alerts.entities.registryValueType + type: keyword + description: > + Value type of the registry key/value pair related to the event. + - name: alerts.entities.deviceId + type: keyword + description: > + The unique ID of the device related to the event. + - name: alerts.entities.ipAddress + type: keyword + description: > + The related IP address to the event. + - name: alerts.devices + type: flattened + description: > + The devices related to the investigation. + diff --git a/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml new file mode 100644 index 00000000000..2b2a6f936f7 --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/config/defender.yml @@ -0,0 +1,43 @@ +{{ if eq .input "httpjson" }} + +type: httpjson +http_method: GET +interval: {{ .interval }} +json_objects_array: value +split_events_by: alerts..entities +url: {{ .url }} + +oauth2: {{ .oauth2 | tojson }} +oauth2.provider: azure +oauth2.azure.resource: https://api.security.microsoft.com +http_headers: + User-Agent: MdatpPartner-Elastic-Filebeat/1.0.0 +date_cursor.field: lastUpdateTime +date_cursor.url_field: '$filter' +date_cursor.value_template: 'lastUpdateTime gt {{.}}' +date_cursor.initial_interval: 55m +date_cursor.date_format: '2006-01-02T15:04:05.9999999Z' + + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{ .tags | tojson }} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: json + - add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml new file mode 100644 index 00000000000..f1ea7c03abd --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/ingest/pipeline.yml @@ -0,0 +1,301 @@ +--- +description: Pipeline for parsing microsoft atp logs +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- remove: + field: + - message + - json.comments + - host + ignore_missing: true + +######################### +## ECS General Mapping ## +######################### +- script: + lang: painless + if: ctx?.json != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + +- set: + field: cloud.provider + value: azure +- set: + field: '@timestamp' + value: '{{json.lastUpdateTime}}' + if: ctx.json?.lastUpdateTime != null +- rename: + field: json.alerts.title + target_field: message + ignore_missing: true + +####################### +## ECS Event Mapping ## +####################### +- set: + field: event.kind + value: alert +# Events returned from the API is always in UTC, so should never use anything else +- set: + field: event.timezone + value: UTC +- set: + field: event.action + value: '{{json.alerts.category}}' + if: ctx.json?.alerts?.category != null +- set: + field: event.provider + value: '{{json.alerts.serviceSource}}' + if: ctx.json?.alerts?.serviceSource != null +- set: + field: event.created + value: '{{json.createdTime}}' + if: ctx.json?.createdTime != null +- append: + field: event.category + value: host +- append: + field: event.category + value: malware + if: ctx.json?.determination == 'Malware' +- append: + field: event.category + value: process + if: ctx.json?.entities?.entityType == 'Process' +- append: + field: event.type + value: user + if: ctx.json?.entities?.entityType == 'User' +- append: + field: event.type + value: + - creation + - start + if: ctx.json?.status == 'New' +- append: + field: event.type + value: end + if: ctx.json?.status == 'Resolved' +- rename: + field: json.alerts.alertId + target_field: event.id + ignore_missing: true +- rename: + field: json.alerts.firstActivity + target_field: event.start + ignore_missing: true +- rename: + field: json.alerts.lastActivity + target_field: event.end + ignore_missing: true +- set: + field: event.severity + value: 0 + if: ctx.json?.severity == 'Unspecified' +- set: + field: event.severity + value: 1 + if: ctx.json?.severity == 'Informational' +- set: + field: event.severity + value: 2 + if: ctx.json?.severity == 'Low' +- set: + field: event.severity + value: 3 + if: ctx.json?.severity == 'Medium' +- set: + field: event.severity + value: 4 + if: ctx.json?.severity == 'High' +- script: + lang: painless + if: "ctx?.event?.start != null && ctx?.event?.end != null" + source: > + Instant eventstart = ZonedDateTime.parse(ctx?.event?.start).toInstant(); + Instant eventend = ZonedDateTime.parse(ctx?.event?.end).toInstant(); + ctx.event['duration'] = ChronoUnit.NANOS.between(eventstart, eventend); + +######################## +## ECS Threat Mapping ## +######################## +- set: + field: threat.framework + value: MITRE ATT&CK + if: ctx.json?.alerts?.category != null +- rename: + field: json.alerts.category + target_field: threat.technique.name + ignore_missing: true +- rename: + field: json.alerts.description + target_field: rule.description + ignore_missing: true + if: ctx.json?.alerts?.description.length() < 1020 + +###################### +## ECS File Mapping ## +###################### +- rename: + field: json.alerts.entities.fileName + target_field: file.name + ignore_missing: true +- rename: + field: json.alerts.entities.sha256 + target_field: file.hash.sha256 + ignore_missing: true +- rename: + field: json.alerts.entities.sha1 + target_field: file.hash.sha1 + ignore_missing: true +- rename: + field: json.alerts.entities.filePath + target_field: file.path + ignore_missing: true + +######################### +## ECS Process Mapping ## +######################### +- rename: + field: json.alerts.entities.processId + target_field: process.pid + ignore_missing: true +- rename: + field: json.alerts.entities.processCommandLine + target_field: process.command_line + ignore_missing: true +- rename: + field: json.alerts.entities.processCreationTime + target_field: process.start + ignore_missing: true +- rename: + field: json.alerts.entities.parentProcessId + target_field: process.parent.pid + ignore_missing: true +- rename: + field: json.alerts.entities.parentProcessCreationTime + target_field: process.parent.start + ignore_missing: true + +########################## +## ECS Observer Mapping ## +########################## +- set: + field: observer.product + value: 365 Defender +- set: + field: observer.vendor + value: Microsoft +- rename: + field: json.alerts.serviceSource + target_field: observer.name + ignore_missing: true + +##################### +## ECS URL Mapping ## +##################### +- rename: + field: json.alerts.entities.url + target_field: url.full + ignore_missing: true + if: ctx?.json?.entities?.url != null + +###################### +## ECS User Mapping ## +###################### +- rename: + field: json.alerts.entities.userPrincipalName + target_field: host.user.name + ignore_missing: true +- rename: + field: json.alerts.entities.domainName + target_field: host.user.domain + ignore_missing: true +- rename: + field: json.alerts.entities.aadUserId + target_field: host.user.id + ignore_missing: true + +######################### +## ECS Related Mapping ## +######################### +- append: + field: related.ip + value: '{{json.alerts.entities.ipAddress}}' + if: ctx.json?.entities?.ipAddress != null +- append: + field: related.user + value: '{{host.user.name}}' + if: ctx.host?.user?.name != null +- append: + field: related.hash + value: '{{file.hash.sha1}}' + if: ctx.file?.hash?.sha1 != null +- append: + field: related.hash + value: '{{file.hash.sha256}}' + if: ctx.file?.hash?.sha256 != null +- append: + field: related.hosts + value: '{{host.hostname}}' + if: ctx.host?.hostname != null + +############# +## Cleanup ## +############# +- convert: + field: json.alerts.incidentId + type: string + ignore_missing: true +- convert: + field: json.incidentId + type: string + ignore_missing: true +- remove: + field: json.alerts.mitreTechniques + ignore_missing: true + if: ctx?.json?.alerts?.mitreTechniques.isEmpty() +- remove: + field: json.alerts.devices + ignore_missing: true + if: ctx?.json?.alerts?.devices.isEmpty() +- remove: + field: json.tags + ignore_missing: true + if: ctx?.json?.tags.isEmpty() +- remove: + ignore_missing: true + field: + - json.createdTime + - json.severity + - json.lastUpdateTime +- rename: + field: json + target_field: microsoft.m365_defender + ignore_missing: true +on_failure: +- set: + field: error.message + value: '{{_ingest.on_failure_message}}' diff --git a/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml new file mode 100644 index 00000000000..d7b73352f79 --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/manifest.yml @@ -0,0 +1,17 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: interval + default: 5m + - name: tags + default: [m365-defender, forwarded] + - name: url + default: "https://api.security.microsoft.com/api/incidents" + - name: oauth2 + +ingest_pipeline: ingest/pipeline.yml +input: config/defender.yml + + diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log new file mode 100644 index 00000000000..4fc4cf141b6 --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log @@ -0,0 +1,9 @@ +{"status":"Resolved","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low","alerts":{"creationTime":"2020-06-30T09:32:31.4579225Z","detectionSource":"WindowsDefenderAv","firstActivity":"2020-06-30T09:31:22.5729558Z","incidentId":12,"serviceSource":"MicrosoftDefenderATP","actorName":null,"alertId":"da637291063515066999_-2102938302","determination":null,"lastActivity":"2020-06-30T09:46:15.0876676Z","assignedTo":"Automation","devices":[{"osBuild":17763,"osProcessor":"x64","rbacGroupId":0,"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"riskScore":"High","version":"Other","deviceDnsName":"TestServer4","healthStatus":"Inactive","osPlatform":"Other"}],"investigationId":9,"threatFamilyName":null,"title":"'Mountsi' malware was detected","category":"Malware","classification":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"amsistream-1D89ECED25A52AB98B76FF619B7BA07A","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356"},"investigationState":"Benign","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","severity":"Informational","status":"Resolved"},"assignedTo":"elastic@elasticuser.com","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[]} +{"incidentName":"12","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"devices":[{"aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"deviceDnsName":"TestServer4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"osProcessor":"x64","riskScore":"High","version":"Other"}],"entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"amsistream-B103C1A335BDB049E617D1AC4A41FCDC","sha1":"ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281","sha256":"fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356"},"firstActivity":"2020-06-30T09:31:22.5729558Z","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","alertId":"da637291063515066999_-2102938302","category":"Malware","classification":null,"determination":null,"mitreTechniques":[],"serviceSource":"MicrosoftDefenderATP","status":"Resolved","assignedTo":"Automation","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":9,"severity":"Informational","title":"'Mountsi' malware was detected","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","lastActivity":"2020-06-30T09:46:15.0876676Z","resolvedTime":"2020-06-30T11:13:12.2680434Z","creationTime":"2020-06-30T09:32:31.4579225Z","investigationState":"Benign","threatFamilyName":null},"assignedTo":"elastic@elasticuser.com","determination":"NotAvailable","incidentId":12,"lastUpdateTime":"2020-09-23T19:44:36.29Z","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z"} +{"alerts":{"mitreTechniques":[],"status":"Resolved","title":"'Exeselrun' malware was detected","threatFamilyName":null,"alertId":"da637291085389812387_-1296910232","category":"Malware","detectionSource":"WindowsDefenderAv","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","serviceSource":"MicrosoftDefenderATP","severity":"Informational","resolvedTime":"2020-06-30T11:13:12.2680434Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.","devices":[{"deviceDnsName":"testserver4","healthStatus":"Inactive","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"osPlatform":"Other","rbacGroupName":null}],"firstActivity":"2020-06-30T10:07:44.3144099Z","incidentId":12,"investigationId":9,"investigationState":"Benign","lastActivity":"2020-06-30T10:07:44.3144099Z","actorName":null,"assignedTo":"Automation","classification":null,"creationTime":"2020-06-30T10:08:58.9655663Z","determination":null,"entities":{"fileName":"SB.xsl","filePath":"C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5","sha1":"d1bb29ce3d01d01451e3623132545d5f577a1bd6","sha256":"ce8d3a3811a3bf923902d6924532308506fe5d024435ddee0cabf90ad9b29f6a","deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File"}},"createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentId":12,"lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown","incidentName":"12","severity":"Low","status":"Resolved","tags":[]} +{"classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","determination":"NotAvailable","incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"severity":"Low","assignedTo":"elastic@elasticuser.com","status":"Resolved","incidentId":12,"tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","firstActivity":"2020-06-30T10:07:44.333733Z","investigationId":9,"mitreTechniques":[],"resolvedTime":"2020-06-30T11:13:12.2680434Z","title":"An active 'Exeselrun' malware was detected","alertId":"da637291085411733957_-1043898914","category":"Malware","classification":null,"detectionSource":"WindowsDefenderAv","determination":null,"threatFamilyName":null,"actorName":null,"serviceSource":"MicrosoftDefenderATP","status":"Resolved","creationTime":"2020-06-30T10:09:01.1569718Z","devices":[{"deviceDnsName":"TestServer4","healthStatus":"Inactive","osBuild":17763,"osProcessor":"x64","rbacGroupName":null,"riskScore":"High","version":"Other","aadDeviceId":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osPlatform":"Other","rbacGroupId":0}],"entities":{"filePath":"C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5","deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"File","fileName":"SB.xsl"},"incidentId":12,"investigationState":"Benign","lastActivity":"2020-06-30T10:07:44.333733Z","lastUpdatedTime":"2020-08-26T09:41:27.7233333Z","severity":"Low","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection."}} +{"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","redirectIncidentId":null,"severity":"Low","status":"Resolved","tags":[],"alerts":{"assignedTo":"elastic@elasticuser.com","determination":null,"serviceSource":"MicrosoftDefenderATP","severity":"Low","alertId":"da637291086161511365_-2075772905","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:31:04.1092404Z","processId":6720},"mitreTechniques":[],"title":"Suspicious 'AccessibilityEscalation' behavior was detected","category":"SuspiciousActivity","devices":[{"aadDeviceId":null,"mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osProcessor":"x64","riskScore":"High","osPlatform":"Other","rbacGroupId":0,"rbacGroupName":null,"version":"Other","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","status":"Resolved","detectionSource":"WindowsDefenderAv","incidentId":12,"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","resolvedTime":"2020-09-23T19:44:36.1092821Z","threatFamilyName":null,"actorName":null},"determination":"NotAvailable","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z"} +{"determination":"NotAvailable","severity":"Low","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","incidentId":12,"incidentName":"12","lastUpdateTime":"2020-09-23T19:44:36.29Z","redirectIncidentId":null,"alerts":{"lastActivity":"2020-06-30T10:31:09.4165785Z","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","actorName":null,"description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","determination":null,"entities":{"accountName":"","entityType":"User"},"firstActivity":"2020-06-30T10:09:10.8889583Z","investigationState":"UnsupportedAlertType","serviceSource":"MicrosoftDefenderATP","status":"Resolved","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","devices":[{"aadDeviceId":null,"healthStatus":"Inactive","osPlatform":"Other","osProcessor":"x64","riskScore":"High","deviceDnsName":"testserver4","firstSeen":"2020-06-30T08:55:08.8320449Z","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","osBuild":17763,"rbacGroupId":0,"rbacGroupName":null,"version":"Other"}],"mitreTechniques":[],"severity":"Low","threatFamilyName":null,"creationTime":"2020-06-30T10:10:16.1355657Z","detectionSource":"WindowsDefenderAv","incidentId":12,"alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","category":"SuspiciousActivity","investigationId":null,"resolvedTime":"2020-09-23T19:44:36.1092821Z"},"assignedTo":"elastic@elasticuser.com","status":"Resolved","tags":[]} +{"determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:44:36.29Z","tags":[],"alerts":{"investigationState":"UnsupportedAlertType","status":"Resolved","alertId":"da637291086161511365_-2075772905","assignedTo":"elastic@elasticuser.com","determination":null,"firstActivity":"2020-06-30T10:09:10.8889583Z","mitreTechniques":[],"resolvedTime":"2020-09-23T19:44:36.1092821Z","severity":"Low","actorName":null,"category":"SuspiciousActivity","description":"Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.","lastUpdatedTime":"2020-09-23T19:44:37.9666667Z","title":"Suspicious 'AccessibilityEscalation' behavior was detected","classification":"FalsePositive","creationTime":"2020-06-30T10:10:16.1355657Z","entities":{"deviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","entityType":"Process","processCreationTime":"2020-06-30T10:09:10.5747992Z","processId":1324},"incidentId":12,"serviceSource":"MicrosoftDefenderATP","threatFamilyName":null,"detectionSource":"WindowsDefenderAv","devices":[{"osPlatform":"Other","osProcessor":"x64","rbacGroupId":0,"riskScore":"High","version":"Other","aadDeviceId":null,"deviceDnsName":"testserver4","mdatpDeviceId":"75a63a39f9bc5a964f417c11f6277d5bf9489f0d","rbacGroupName":null,"firstSeen":"2020-06-30T08:55:08.8320449Z","healthStatus":"Inactive","osBuild":17763}],"investigationId":null,"lastActivity":"2020-06-30T10:31:09.4165785Z"},"assignedTo":"elastic@elasticuser.com","classification":"Unknown","createdTime":"2020-06-30T09:32:31.85Z","status":"Resolved","incidentId":12,"incidentName":"12","redirectIncidentId":null,"severity":"Low"} +{"incidentId":14,"incidentName":"Activity from infrequent country","redirectIncidentId":null,"tags":[],"alerts":{"category":"SuspiciousActivity","entities":{"aadUserId":"8e24c50a-a77c-4782-813f-965009b5ddf3","accountName":"brent","entityType":"User","userPrincipalName":"brent@elasticbv.onmicrosoft.com"},"incidentId":14,"investigationState":"UnsupportedAlertType","status":"New","actorName":null,"classification":"FalsePositive","description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","investigationId":null,"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","mitreTechniques":[],"serviceSource":"MicrosoftCloudAppSecurity","severity":"Medium","threatFamilyName":null,"title":"Activity from infrequent country","assignedTo":"elastic@elasticuser.com","detectionSource":"MCAS","devices":[],"alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","creationTime":"2020-07-27T15:54:20.52207Z","determination":null,"firstActivity":"2020-07-27T15:47:22.088Z","resolvedTime":null},"classification":"Unknown","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","severity":"Medium","status":"Active","assignedTo":"elastic@elasticuser.com","createdTime":"2020-07-27T15:54:21.58Z"} +{"incidentId":14,"incidentName":"Activity from infrequent country","severity":"Medium","status":"Active","tags":[],"alerts":{"description":"Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.","detectionSource":"MCAS","firstActivity":"2020-07-27T15:47:22.088Z","investigationId":null,"investigationState":"UnsupportedAlertType","severity":"Medium","alertId":"caA214771F-6AB0-311D-B2B0-BECD3B4A967B","category":"SuspiciousActivity","classification":"FalsePositive","determination":null,"entities":{"entityType":"Ip","ipAddress":"73.172.171.53"},"incidentId":14,"serviceSource":"MicrosoftCloudAppSecurity","status":"New","actorName":null,"title":"Activity from infrequent country","devices":[],"lastActivity":"2020-07-27T15:47:22.088Z","lastUpdatedTime":"2020-09-23T19:32:17.5433333Z","creationTime":"2020-07-27T15:54:20.52207Z","mitreTechniques":[],"resolvedTime":null,"threatFamilyName":null,"assignedTo":"elastic@elasticuser.com"},"createdTime":"2020-07-27T15:54:21.58Z","determination":"NotAvailable","lastUpdateTime":"2020-09-23T19:32:05.8366667Z","redirectIncidentId":null,"assignedTo":"elastic@elasticuser.com","classification":"Unknown"} diff --git a/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json new file mode 100644 index 00000000000..1f81a57a98f --- /dev/null +++ b/x-pack/filebeat/module/microsoft/m365_defender/test/m365_defender-test.ndjson.log-expected.json @@ -0,0 +1,611 @@ +[ + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "Malware", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 892514711800, + "event.end": "2020-06-30T09:46:15.0876676Z", + "event.id": "da637291063515066999_-2102938302", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T09:31:22.5729558Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "file.hash.sha1": "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281", + "file.hash.sha256": "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356", + "file.name": "amsistream-1D89ECED25A52AB98B76FF619B7BA07A", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 0, + "message": "'Mountsi' malware was detected", + "microsoft.m365_defender.alerts.assignedTo": "Automation", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T09:32:31.4579225Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "TestServer4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "File", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationId": 9, + "microsoft.m365_defender.alerts.investigationState": "Benign", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z", + "microsoft.m365_defender.alerts.severity": "Informational", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "related.hash": [ + "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281", + "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356" + ], + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "Malware" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "Malware", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 892514711800, + "event.end": "2020-06-30T09:46:15.0876676Z", + "event.id": "da637291063515066999_-2102938302", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T09:31:22.5729558Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "file.hash.sha1": "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281", + "file.hash.sha256": "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356", + "file.name": "amsistream-B103C1A335BDB049E617D1AC4A41FCDC", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 2071, + "message": "'Mountsi' malware was detected", + "microsoft.m365_defender.alerts.assignedTo": "Automation", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T09:32:31.4579225Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "TestServer4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "File", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationId": 9, + "microsoft.m365_defender.alerts.investigationState": "Benign", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z", + "microsoft.m365_defender.alerts.severity": "Informational", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "related.hash": [ + "ffb1670c6c6a9c5b4c5cea8b6b8e68d62e7ff281", + "fd46705c4f67a8ef16e76259ca6d6253241e51a1f8952223145f92aa1907d356" + ], + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "Malware" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "Malware", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 0, + "event.end": "2020-06-30T10:07:44.3144099Z", + "event.id": "da637291085389812387_-1296910232", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T10:07:44.3144099Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "file.hash.sha1": "d1bb29ce3d01d01451e3623132545d5f577a1bd6", + "file.hash.sha256": "ce8d3a3811a3bf923902d6924532308506fe5d024435ddee0cabf90ad9b29f6a", + "file.name": "SB.xsl", + "file.path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 4142, + "message": "'Exeselrun' malware was detected", + "microsoft.m365_defender.alerts.assignedTo": "Automation", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:08:58.9655663Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "testserver4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "File", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationId": 9, + "microsoft.m365_defender.alerts.investigationState": "Benign", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z", + "microsoft.m365_defender.alerts.severity": "Informational", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "related.hash": [ + "d1bb29ce3d01d01451e3623132545d5f577a1bd6", + "ce8d3a3811a3bf923902d6924532308506fe5d024435ddee0cabf90ad9b29f6a" + ], + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nThis detection might indicate that the malware was stopped from delivering its payload. However, it is prudent to check the machine for signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "Malware" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "Malware", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 0, + "event.end": "2020-06-30T10:07:44.333733Z", + "event.id": "da637291085411733957_-1043898914", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T10:07:44.333733Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "file.name": "SB.xsl", + "file.path": "C:\\Windows\\Temp\\sb-sim-temp-ikyxqi\\sb_10554_bs_h4qpk5", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 6249, + "message": "An active 'Exeselrun' malware was detected", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:09:01.1569718Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "TestServer4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "File", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationId": 9, + "microsoft.m365_defender.alerts.investigationState": "Benign", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-08-26T09:41:27.7233333Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-06-30T11:13:12.2680434Z", + "microsoft.m365_defender.alerts.severity": "Low", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "Malware" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "SuspiciousActivity", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 1318527620200, + "event.end": "2020-06-30T10:31:09.4165785Z", + "event.id": "da637291086161511365_-2075772905", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T10:09:10.8889583Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 8376, + "message": "Suspicious 'AccessibilityEscalation' behavior was detected", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.classification": "FalsePositive", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:10:16.1355657Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "testserver4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "Process", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:44:37.9666667Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-09-23T19:44:36.1092821Z", + "microsoft.m365_defender.alerts.severity": "Low", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "process.pid": 6720, + "process.start": "2020-06-30T10:31:04.1092404Z", + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "SuspiciousActivity" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "SuspiciousActivity", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 1318527620200, + "event.end": "2020-06-30T10:31:09.4165785Z", + "event.id": "da637291086161511365_-2075772905", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T10:09:10.8889583Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 10542, + "message": "Suspicious 'AccessibilityEscalation' behavior was detected", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.classification": "FalsePositive", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:10:16.1355657Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "testserver4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.accountName": "", + "microsoft.m365_defender.alerts.entities.entityType": "User", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:44:37.9666667Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-09-23T19:44:36.1092821Z", + "microsoft.m365_defender.alerts.severity": "Low", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "SuspiciousActivity" + }, + { + "@timestamp": "2020-09-23T19:44:36.29Z", + "cloud.provider": "azure", + "event.action": "SuspiciousActivity", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 1318527620200, + "event.end": "2020-06-30T10:31:09.4165785Z", + "event.id": "da637291086161511365_-2075772905", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftDefenderATP", + "event.severity": 2, + "event.start": "2020-06-30T10:09:10.8889583Z", + "event.timezone": "UTC", + "event.type": [ + "end" + ], + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 12598, + "message": "Suspicious 'AccessibilityEscalation' behavior was detected", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.classification": "FalsePositive", + "microsoft.m365_defender.alerts.creationTime": "2020-06-30T10:10:16.1355657Z", + "microsoft.m365_defender.alerts.detectionSource": "WindowsDefenderAv", + "microsoft.m365_defender.alerts.devices": [ + { + "deviceDnsName": "testserver4", + "firstSeen": "2020-06-30T08:55:08.8320449Z", + "healthStatus": "Inactive", + "mdatpDeviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "osBuild": 17763, + "osPlatform": "Other", + "osProcessor": "x64", + "rbacGroupId": 0, + "riskScore": "High", + "version": "Other" + } + ], + "microsoft.m365_defender.alerts.entities.deviceId": "75a63a39f9bc5a964f417c11f6277d5bf9489f0d", + "microsoft.m365_defender.alerts.entities.entityType": "Process", + "microsoft.m365_defender.alerts.incidentId": "12", + "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:44:37.9666667Z", + "microsoft.m365_defender.alerts.resolvedTime": "2020-09-23T19:44:36.1092821Z", + "microsoft.m365_defender.alerts.severity": "Low", + "microsoft.m365_defender.alerts.status": "Resolved", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "12", + "microsoft.m365_defender.incidentName": "12", + "microsoft.m365_defender.status": "Resolved", + "observer.name": "MicrosoftDefenderATP", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "process.pid": 1324, + "process.start": "2020-06-30T10:09:10.5747992Z", + "rule.description": "Malware and unwanted software are undesirable applications that perform annoying, disruptive, or harmful actions on affected machines. Some of these undesirable applications can replicate and spread from one machine to another. Others are able to receive commands from remote attackers and perform activities associated with cyber attacks.\n\nA malware is considered active if it is found running on the machine or it already has persistence mechanisms in place. Active malware detections are assigned higher severity ratings.\n\nBecause this malware was active, take precautionary measures and check for residual signs of infection.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "SuspiciousActivity" + }, + { + "@timestamp": "2020-09-23T19:32:05.8366667Z", + "cloud.provider": "azure", + "event.action": "SuspiciousActivity", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 0, + "event.end": "2020-07-27T15:47:22.088Z", + "event.id": "caA214771F-6AB0-311D-B2B0-BECD3B4A967B", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftCloudAppSecurity", + "event.severity": 3, + "event.start": "2020-07-27T15:47:22.088Z", + "event.timezone": "UTC", + "fileset.name": "m365_defender", + "host.user.id": "8e24c50a-a77c-4782-813f-965009b5ddf3", + "host.user.name": "brent@elasticbv.onmicrosoft.com", + "input.type": "log", + "log.offset": 14764, + "message": "Activity from infrequent country", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.classification": "FalsePositive", + "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z", + "microsoft.m365_defender.alerts.detectionSource": "MCAS", + "microsoft.m365_defender.alerts.entities.accountName": "brent", + "microsoft.m365_defender.alerts.entities.entityType": "User", + "microsoft.m365_defender.alerts.incidentId": "14", + "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z", + "microsoft.m365_defender.alerts.severity": "Medium", + "microsoft.m365_defender.alerts.status": "New", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "14", + "microsoft.m365_defender.incidentName": "Activity from infrequent country", + "microsoft.m365_defender.status": "Active", + "observer.name": "MicrosoftCloudAppSecurity", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "related.user": [ + "brent@elasticbv.onmicrosoft.com" + ], + "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "SuspiciousActivity" + }, + { + "@timestamp": "2020-09-23T19:32:05.8366667Z", + "cloud.provider": "azure", + "event.action": "SuspiciousActivity", + "event.category": [ + "host" + ], + "event.dataset": "microsoft.m365_defender", + "event.duration": 0, + "event.end": "2020-07-27T15:47:22.088Z", + "event.id": "caA214771F-6AB0-311D-B2B0-BECD3B4A967B", + "event.kind": "alert", + "event.module": "microsoft", + "event.provider": "MicrosoftCloudAppSecurity", + "event.severity": 3, + "event.start": "2020-07-27T15:47:22.088Z", + "event.timezone": "UTC", + "fileset.name": "m365_defender", + "input.type": "log", + "log.offset": 16091, + "message": "Activity from infrequent country", + "microsoft.m365_defender.alerts.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.alerts.classification": "FalsePositive", + "microsoft.m365_defender.alerts.creationTime": "2020-07-27T15:54:20.52207Z", + "microsoft.m365_defender.alerts.detectionSource": "MCAS", + "microsoft.m365_defender.alerts.entities.entityType": "Ip", + "microsoft.m365_defender.alerts.entities.ipAddress": "73.172.171.53", + "microsoft.m365_defender.alerts.incidentId": "14", + "microsoft.m365_defender.alerts.investigationState": "UnsupportedAlertType", + "microsoft.m365_defender.alerts.lastUpdatedTime": "2020-09-23T19:32:17.5433333Z", + "microsoft.m365_defender.alerts.severity": "Medium", + "microsoft.m365_defender.alerts.status": "New", + "microsoft.m365_defender.assignedTo": "elastic@elasticuser.com", + "microsoft.m365_defender.classification": "Unknown", + "microsoft.m365_defender.determination": "NotAvailable", + "microsoft.m365_defender.incidentId": "14", + "microsoft.m365_defender.incidentName": "Activity from infrequent country", + "microsoft.m365_defender.status": "Active", + "observer.name": "MicrosoftCloudAppSecurity", + "observer.product": "365 Defender", + "observer.vendor": "Microsoft", + "rule.description": "Brent Murphy (brent@elasticbv.onmicrosoft.com) performed an activity. No activity was performed in United States in the past 41 days.", + "service.type": "microsoft", + "tags": [ + "m365-defender", + "forwarded" + ], + "threat.framework": "MITRE ATT&CK", + "threat.technique.name": "SuspiciousActivity" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/misp/threat/config/input.yml b/x-pack/filebeat/module/misp/threat/config/input.yml index 3ff985b07f3..26010fc478a 100644 --- a/x-pack/filebeat/module/misp/threat/config/input.yml +++ b/x-pack/filebeat/module/misp/threat/config/input.yml @@ -37,4 +37,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/mssql/log/config/config.yml b/x-pack/filebeat/module/mssql/log/config/config.yml index 31990fb32c3..0ac3d9da2c2 100644 --- a/x-pack/filebeat/module/mssql/log/config/config.yml +++ b/x-pack/filebeat/module/mssql/log/config/config.yml @@ -14,4 +14,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/netflow/log/config/netflow.yml b/x-pack/filebeat/module/netflow/log/config/netflow.yml index b34160bd6b9..f56ffeed226 100644 --- a/x-pack/filebeat/module/netflow/log/config/netflow.yml +++ b/x-pack/filebeat/module/netflow/log/config/netflow.yml @@ -31,4 +31,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/o365/audit/config/input.yml b/x-pack/filebeat/module/o365/audit/config/input.yml index d41a5bb9aab..ec30daef127 100644 --- a/x-pack/filebeat/module/o365/audit/config/input.yml +++ b/x-pack/filebeat/module/o365/audit/config/input.yml @@ -62,4 +62,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index a544ab8dc65..487dfdf165e 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/rabbitmq/log/config/log.yml b/x-pack/filebeat/module/rabbitmq/log/config/log.yml index bc46f2458c8..1f32bbf71cf 100644 --- a/x-pack/filebeat/module/rabbitmq/log/config/log.yml +++ b/x-pack/filebeat/module/rabbitmq/log/config/log.yml @@ -18,4 +18,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/sophos/xg/config/config.yml b/x-pack/filebeat/module/sophos/xg/config/config.yml index 86c12e9ec08..8585ff4a19c 100644 --- a/x-pack/filebeat/module/sophos/xg/config/config.yml +++ b/x-pack/filebeat/module/sophos/xg/config/config.yml @@ -27,7 +27,7 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 - add_fields: target: '_conf' fields: diff --git a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml index 6b6fcf216f2..5794bd0ca6f 100644 --- a/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml +++ b/x-pack/filebeat/module/zeek/capture_loss/config/capture_loss.yml @@ -22,4 +22,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/connection/config/connection.yml b/x-pack/filebeat/module/zeek/connection/config/connection.yml index 8a79295724f..f6caa143689 100644 --- a/x-pack/filebeat/module/zeek/connection/config/connection.yml +++ b/x-pack/filebeat/module/zeek/connection/config/connection.yml @@ -102,4 +102,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml index 45010e08973..7d7f0c87353 100644 --- a/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml +++ b/x-pack/filebeat/module/zeek/dce_rpc/config/dce_rpc.yml @@ -58,4 +58,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml index f1a2f0ced3a..b049d571c14 100644 --- a/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml +++ b/x-pack/filebeat/module/zeek/dhcp/config/dhcp.yml @@ -120,4 +120,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml index 7730d2b6d85..67033a42cf2 100644 --- a/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml +++ b/x-pack/filebeat/module/zeek/dnp3/config/dnp3.yml @@ -68,4 +68,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dns/config/dns.yml b/x-pack/filebeat/module/zeek/dns/config/dns.yml index 86a2022d695..9e0f7e952e2 100644 --- a/x-pack/filebeat/module/zeek/dns/config/dns.yml +++ b/x-pack/filebeat/module/zeek/dns/config/dns.yml @@ -203,4 +203,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml index acc6defd4df..67565ef21ab 100644 --- a/x-pack/filebeat/module/zeek/dpd/config/dpd.yml +++ b/x-pack/filebeat/module/zeek/dpd/config/dpd.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/files/config/files.yml b/x-pack/filebeat/module/zeek/files/config/files.yml index 65c067609c9..0e0ce22e2bf 100644 --- a/x-pack/filebeat/module/zeek/files/config/files.yml +++ b/x-pack/filebeat/module/zeek/files/config/files.yml @@ -42,4 +42,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml index 51a3c053576..671a1162c66 100644 --- a/x-pack/filebeat/module/zeek/ftp/config/ftp.yml +++ b/x-pack/filebeat/module/zeek/ftp/config/ftp.yml @@ -86,4 +86,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/http/config/http.yml b/x-pack/filebeat/module/zeek/http/config/http.yml index 4c7c812d0cc..8a43f07f0dd 100644 --- a/x-pack/filebeat/module/zeek/http/config/http.yml +++ b/x-pack/filebeat/module/zeek/http/config/http.yml @@ -93,4 +93,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/intel/config/intel.yml b/x-pack/filebeat/module/zeek/intel/config/intel.yml index 5b73833ea35..80d565745f1 100644 --- a/x-pack/filebeat/module/zeek/intel/config/intel.yml +++ b/x-pack/filebeat/module/zeek/intel/config/intel.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/irc/config/irc.yml b/x-pack/filebeat/module/zeek/irc/config/irc.yml index 54aaa9d4f4b..7e09f9fe376 100644 --- a/x-pack/filebeat/module/zeek/irc/config/irc.yml +++ b/x-pack/filebeat/module/zeek/irc/config/irc.yml @@ -72,4 +72,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml index d656ad0ab6a..a65bb88c0d0 100644 --- a/x-pack/filebeat/module/zeek/modbus/config/modbus.yml +++ b/x-pack/filebeat/module/zeek/modbus/config/modbus.yml @@ -73,4 +73,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml index 4c6e70d9f1c..963c8469fe1 100644 --- a/x-pack/filebeat/module/zeek/mysql/config/mysql.yml +++ b/x-pack/filebeat/module/zeek/mysql/config/mysql.yml @@ -72,4 +72,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/notice/config/notice.yml b/x-pack/filebeat/module/zeek/notice/config/notice.yml index 649d3f3ba97..07ccbfb76b4 100644 --- a/x-pack/filebeat/module/zeek/notice/config/notice.yml +++ b/x-pack/filebeat/module/zeek/notice/config/notice.yml @@ -104,4 +104,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml index c67f66b54b9..135086e19f9 100644 --- a/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml +++ b/x-pack/filebeat/module/zeek/ntlm/config/ntlm.yml @@ -86,4 +86,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml index 874a0fde6d9..933f829b747 100644 --- a/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml +++ b/x-pack/filebeat/module/zeek/ocsp/config/ocsp.yml @@ -64,4 +64,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/pe/config/pe.yml b/x-pack/filebeat/module/zeek/pe/config/pe.yml index 3df430d7dc9..cfb8ea8c4b0 100644 --- a/x-pack/filebeat/module/zeek/pe/config/pe.yml +++ b/x-pack/filebeat/module/zeek/pe/config/pe.yml @@ -33,4 +33,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/radius/config/radius.yml b/x-pack/filebeat/module/zeek/radius/config/radius.yml index 66fccaa3f5c..471275d5ece 100644 --- a/x-pack/filebeat/module/zeek/radius/config/radius.yml +++ b/x-pack/filebeat/module/zeek/radius/config/radius.yml @@ -58,4 +58,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml index de71448fb1b..071cc59fab9 100644 --- a/x-pack/filebeat/module/zeek/rdp/config/rdp.yml +++ b/x-pack/filebeat/module/zeek/rdp/config/rdp.yml @@ -88,4 +88,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml index 3adb14c55bf..417e9ab4dcd 100644 --- a/x-pack/filebeat/module/zeek/rfb/config/rfb.yml +++ b/x-pack/filebeat/module/zeek/rfb/config/rfb.yml @@ -73,4 +73,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/sip/config/sip.yml b/x-pack/filebeat/module/zeek/sip/config/sip.yml index 7aa30034de2..97885a1e2dc 100644 --- a/x-pack/filebeat/module/zeek/sip/config/sip.yml +++ b/x-pack/filebeat/module/zeek/sip/config/sip.yml @@ -95,4 +95,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml index 763379a7d88..885669e4c9c 100644 --- a/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml +++ b/x-pack/filebeat/module/zeek/smb_cmd/config/smb_cmd.yml @@ -101,4 +101,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml index c5f7c2e53e7..a3f02a7558f 100644 --- a/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml +++ b/x-pack/filebeat/module/zeek/smb_files/config/smb_files.yml @@ -61,4 +61,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml index 624454ed171..59a68e2054b 100644 --- a/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml +++ b/x-pack/filebeat/module/zeek/smb_mapping/config/smb_mapping.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml index 5b2f6595df2..0af383a7187 100644 --- a/x-pack/filebeat/module/zeek/smtp/config/smtp.yml +++ b/x-pack/filebeat/module/zeek/smtp/config/smtp.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml index 0c7e05ce6db..1fa024742be 100644 --- a/x-pack/filebeat/module/zeek/snmp/config/snmp.yml +++ b/x-pack/filebeat/module/zeek/snmp/config/snmp.yml @@ -69,4 +69,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/socks/config/socks.yml b/x-pack/filebeat/module/zeek/socks/config/socks.yml index f834e5d1bcc..99613e4bfe3 100644 --- a/x-pack/filebeat/module/zeek/socks/config/socks.yml +++ b/x-pack/filebeat/module/zeek/socks/config/socks.yml @@ -67,4 +67,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml index c855d49dff2..45f160cd99d 100644 --- a/x-pack/filebeat/module/zeek/ssh/config/ssh.yml +++ b/x-pack/filebeat/module/zeek/ssh/config/ssh.yml @@ -76,4 +76,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/stats/config/stats.yml b/x-pack/filebeat/module/zeek/stats/config/stats.yml index cdf243f7a45..45c22b6b3d2 100644 --- a/x-pack/filebeat/module/zeek/stats/config/stats.yml +++ b/x-pack/filebeat/module/zeek/stats/config/stats.yml @@ -97,4 +97,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml index a89601cb717..0d1b8fc05e5 100644 --- a/x-pack/filebeat/module/zeek/syslog/config/syslog.yml +++ b/x-pack/filebeat/module/zeek/syslog/config/syslog.yml @@ -57,4 +57,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml index 13a2a37cc69..741a0d7e454 100644 --- a/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml +++ b/x-pack/filebeat/module/zeek/traceroute/config/traceroute.yml @@ -45,4 +45,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml index ac636e9e7c0..243f83de760 100644 --- a/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml +++ b/x-pack/filebeat/module/zeek/tunnel/config/tunnel.yml @@ -56,4 +56,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zeek/weird/config/weird.yml b/x-pack/filebeat/module/zeek/weird/config/weird.yml index 5807f95927b..4a4bab4dc33 100644 --- a/x-pack/filebeat/module/zeek/weird/config/weird.yml +++ b/x-pack/filebeat/module/zeek/weird/config/weird.yml @@ -56,4 +56,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml index 207da5447e1..ac04136779c 100644 --- a/x-pack/filebeat/module/zoom/webhook/config/webhook.yml +++ b/x-pack/filebeat/module/zoom/webhook/config/webhook.yml @@ -33,4 +33,4 @@ processors: - add_fields: target: '' fields: - ecs.version: 1.5.0 + ecs.version: 1.6.0 diff --git a/x-pack/filebeat/modules.d/juniper.yml.disabled b/x-pack/filebeat/modules.d/juniper.yml.disabled index e3359756d90..6ffe87834a4 100644 --- a/x-pack/filebeat/modules.d/juniper.yml.disabled +++ b/x-pack/filebeat/modules.d/juniper.yml.disabled @@ -39,3 +39,16 @@ # "local" (default) for system timezone. # "+02:00" for GMT+02:00 # var.tz_offset: local + + srx: + enabled: true + + # Set which input to use between tcp, udp (default) or file. + #var.input: udp + + # The interface to listen to syslog traffic. Defaults to + # localhost. Set to 0.0.0.0 to bind to all available interfaces. + #var.syslog_host: localhost + + # The port to listen for syslog traffic. Defaults to 9006. + #var.syslog_port: 9006 diff --git a/x-pack/filebeat/modules.d/microsoft.yml.disabled b/x-pack/filebeat/modules.d/microsoft.yml.disabled index 09c7211e179..63bcc20897a 100644 --- a/x-pack/filebeat/modules.d/microsoft.yml.disabled +++ b/x-pack/filebeat/modules.d/microsoft.yml.disabled @@ -13,7 +13,20 @@ # Oauth Client Secret #var.oauth2.client.secret: "" - + + # Oauth Token URL, should include the tenant ID + #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" + m365_defender: + enabled: true + # How often the API should be polled + #var.interval: 5m + + # Oauth Client ID + #var.oauth2.client.id: "" + + # Oauth Client Secret + #var.oauth2.client.secret: "" + # Oauth Token URL, should include the tenant ID #var.oauth2.token_url: "https://login.microsoftonline.com/TENANT-ID/oauth2/token" dhcp: diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index 8e380b3957b..098746ff606 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -43,3 +43,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/functionbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/functionbeat/docs/fields.asciidoc b/x-pack/functionbeat/docs/fields.asciidoc index 73c93e39a61..7f4aa3d5aaf 100644 --- a/x-pack/functionbeat/docs/fields.asciidoc +++ b/x-pack/functionbeat/docs/fields.asciidoc @@ -216,8 +216,15 @@ type: object [[exported-fields-ecs]] == ECS fields -ECS Fields. +This section defines Elastic Common Schema (ECS) fields—a common set of fields +to be used when storing event data in {es}. + +This is an exhaustive list, and fields listed here are not necessarily used by {beatname_uc}. +The goal of ECS is to enable and encourage users of {es} to normalize their event data, +so that they can better analyze, visualize, and correlate the data represented in their events. + +See the {ecs-ref}[ECS reference] for more information. *`@timestamp`*:: + diff --git a/x-pack/libbeat/common/cloudfoundry/cache.go b/x-pack/libbeat/common/cloudfoundry/cache.go index 22f41f3b23c..0f19818666e 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache.go +++ b/x-pack/libbeat/common/cloudfoundry/cache.go @@ -5,13 +5,16 @@ package cloudfoundry import ( + "crypto/sha1" + "encoding/base64" "fmt" "time" "github.com/cloudfoundry-community/go-cfclient" + "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/libbeat/persistentcache" ) // cfClient interface is provided so unit tests can mock the actual client. @@ -22,65 +25,113 @@ type cfClient interface { // clientCacheWrap wraps the cloudfoundry client to add a cache in front of GetAppByGuid. type clientCacheWrap struct { - cache *common.Cache + cache *persistentcache.PersistentCache client cfClient log *logp.Logger errorTTL time.Duration } // newClientCacheWrap creates a new cache for application data. -func newClientCacheWrap(client cfClient, ttl time.Duration, errorTTL time.Duration, log *logp.Logger) *clientCacheWrap { +func newClientCacheWrap(client cfClient, cacheName string, ttl time.Duration, errorTTL time.Duration, log *logp.Logger) (*clientCacheWrap, error) { + options := persistentcache.Options{ + Timeout: ttl, + } + + name := "cloudfoundry" + if cacheName != "" { + name = name + "-" + sanitizeCacheName(cacheName) + } + + cache, err := persistentcache.New(name, options) + if err != nil { + return nil, fmt.Errorf("creating metadata cache: %w", err) + } + return &clientCacheWrap{ - cache: common.NewCacheWithExpireOnAdd(ttl, 100), + cache: cache, client: client, errorTTL: errorTTL, log: log, - } + }, nil } type appResponse struct { - app *cfclient.App - err error + App AppMeta `json:"a"` + Error cfclient.CloudFoundryError `json:"e,omitempty"` + ErrorMessage string `json:"em,omitempty"` +} + +func (r *appResponse) fromStructs(app cfclient.App, err error) { + if err != nil { + cause := errors.Cause(err) + if cferr, ok := cause.(cfclient.CloudFoundryError); ok { + r.Error = cferr + } + r.ErrorMessage = err.Error() + return + } + r.App = AppMeta{ + Name: app.Name, + Guid: app.Guid, + SpaceName: app.SpaceData.Entity.Name, + SpaceGuid: app.SpaceData.Meta.Guid, + OrgName: app.SpaceData.Entity.OrgData.Entity.Name, + OrgGuid: app.SpaceData.Entity.OrgData.Meta.Guid, + } +} + +func (r *appResponse) toStructs() (*AppMeta, error) { + var empty cfclient.CloudFoundryError + if r.Error != empty { + // Wrapping the error so cfclient.IsAppNotFoundError can identify it + return nil, errors.Wrap(r.Error, r.ErrorMessage) + } + if len(r.ErrorMessage) > 0 { + return nil, errors.New(r.ErrorMessage) + } + return &r.App, nil } // fetchApp uses the cfClient to retrieve an App entity and // stores it in the internal cache -func (c *clientCacheWrap) fetchAppByGuid(guid string) (*cfclient.App, error) { +func (c *clientCacheWrap) fetchAppByGuid(guid string) (*AppMeta, error) { app, err := c.client.GetAppByGuid(guid) - resp := appResponse{ - app: &app, - err: err, - } + var resp appResponse + resp.fromStructs(app, err) timeout := time.Duration(0) if err != nil { // Cache nil, because is what we want to return when there was an error - resp.app = nil timeout = c.errorTTL } - c.cache.PutWithTimeout(guid, &resp, timeout) - return resp.app, resp.err + err = c.cache.PutWithTimeout(guid, resp, timeout) + if err != nil { + return nil, fmt.Errorf("storing app response in cache: %w", err) + } + return resp.toStructs() } // GetApp returns CF Application info, either from the cache or // using the CF client. -func (c *clientCacheWrap) GetAppByGuid(guid string) (*cfclient.App, error) { - cachedResp := c.cache.Get(guid) - if cachedResp == nil { +func (c *clientCacheWrap) GetAppByGuid(guid string) (*AppMeta, error) { + var resp appResponse + err := c.cache.Get(guid, &resp) + if err != nil { return c.fetchAppByGuid(guid) } - resp, ok := cachedResp.(*appResponse) - if !ok { - return nil, fmt.Errorf("error converting cached app response (of type %T), this is likely a bug", cachedResp) - } - return resp.app, resp.err + return resp.toStructs() } -// StartJanitor starts a goroutine that will periodically clean the applications cache. -func (c *clientCacheWrap) StartJanitor(interval time.Duration) { - c.cache.StartJanitor(interval) +// Close release resources associated with this client +func (c *clientCacheWrap) Close() error { + err := c.cache.Close() + if err != nil { + return fmt.Errorf("closing cache: %w", err) + } + return nil } -// StopJanitor stops the goroutine that periodically clean the applications cache. -func (c *clientCacheWrap) StopJanitor() { - c.cache.StopJanitor() +// sanitizeCacheName returns a unique string that can be used safely as part of a file name +func sanitizeCacheName(name string) string { + hash := sha1.Sum([]byte(name)) + return base64.RawURLEncoding.EncodeToString(hash[:]) } diff --git a/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go index f6af11787c9..f799cc0615f 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go +++ b/x-pack/libbeat/common/cloudfoundry/cache_integration_test.go @@ -31,7 +31,7 @@ func TestGetApps(t *testing.T) { client, err := hub.Client() require.NoError(t, err) - apps, err := client.(*clientCacheWrap).client.(*cfclient.Client).ListApps() + apps, err := client.ListApps() require.NoError(t, err) t.Logf("%d applications available", len(apps)) @@ -40,8 +40,9 @@ func TestGetApps(t *testing.T) { if len(apps) == 0 { t.Skip("no apps in account?") } - client, err := hub.Client() + client, err := hub.ClientWithCache() require.NoError(t, err) + defer client.Close() guid := apps[0].Guid app, err := client.GetAppByGuid(guid) @@ -50,13 +51,15 @@ func TestGetApps(t *testing.T) { }) t.Run("handle error when application is not available", func(t *testing.T) { - client, err := hub.Client() + client, err := hub.ClientWithCache() require.NoError(t, err) + defer client.Close() testNotExists := func(t *testing.T) { app, err := client.GetAppByGuid("notexists") assert.Nil(t, app) - assert.True(t, cfclient.IsAppNotFoundError(err)) + assert.Error(t, err) + assert.True(t, cfclient.IsAppNotFoundError(err), "Error found: %v", err) } var firstTimeDuration time.Duration diff --git a/x-pack/libbeat/common/cloudfoundry/cache_test.go b/x-pack/libbeat/common/cloudfoundry/cache_test.go index 9e18a5ac86e..b345beff226 100644 --- a/x-pack/libbeat/common/cloudfoundry/cache_test.go +++ b/x-pack/libbeat/common/cloudfoundry/cache_test.go @@ -13,19 +13,25 @@ import ( "github.com/cloudfoundry-community/go-cfclient" "github.com/gofrs/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/logp" ) func TestClientCacheWrap(t *testing.T) { - ttl := 500 * time.Millisecond + if testing.Short() { + t.Skip("skipping in short mode") + } + + ttl := 2 * time.Second guid := mustCreateFakeGuid() app := cfclient.App{ - Guid: guid, - Memory: 1, // use this field to track if from cache or from client + Guid: guid, + Name: "Foo", // use this field to track if from cache or from client } fakeClient := &fakeCFClient{app, 0} - cache := newClientCacheWrap(fakeClient, ttl, ttl, logp.NewLogger("cloudfoundry")) + cache, err := newClientCacheWrap(fakeClient, "test", ttl, ttl, logp.NewLogger("cloudfoundry")) + require.NoError(t, err) missingAppGuid := mustCreateFakeGuid() @@ -44,25 +50,28 @@ func TestClientCacheWrap(t *testing.T) { // fetched from client for the first time one, err = cache.GetAppByGuid(guid) assert.NoError(t, err) - assert.Equal(t, app, *one) + assert.Equal(t, app.Guid, one.Guid) + assert.Equal(t, app.Name, one.Name) assert.Equal(t, 2, fakeClient.callCount) // updated app in fake client, new fetch should not have updated app updatedApp := cfclient.App{ - Guid: guid, - Memory: 2, + Guid: guid, + Name: "Bar", } fakeClient.app = updatedApp two, err := cache.GetAppByGuid(guid) assert.NoError(t, err) - assert.Equal(t, app, *two) + assert.Equal(t, app.Guid, two.Guid) + assert.Equal(t, app.Name, two.Name) assert.Equal(t, 2, fakeClient.callCount) // wait the ttl, then it should have updated app time.Sleep(ttl) three, err := cache.GetAppByGuid(guid) assert.NoError(t, err) - assert.Equal(t, updatedApp, *three) + assert.Equal(t, updatedApp.Guid, three.Guid) + assert.Equal(t, updatedApp.Name, three.Name) assert.Equal(t, 3, fakeClient.callCount) } diff --git a/x-pack/libbeat/common/cloudfoundry/config.go b/x-pack/libbeat/common/cloudfoundry/config.go index 0724bdc66e1..2f15d0c7cf9 100644 --- a/x-pack/libbeat/common/cloudfoundry/config.go +++ b/x-pack/libbeat/common/cloudfoundry/config.go @@ -10,8 +10,6 @@ import ( "strings" "time" - "github.com/gofrs/uuid" - "github.com/elastic/beats/v7/libbeat/common/transport/tlscommon" ) @@ -32,14 +30,14 @@ type Config struct { TLS *tlscommon.Config `config:"ssl"` // Override URLs returned from the CF client - APIAddress string `config:"api_address"` + APIAddress string `config:"api_address" validate:"required"` DopplerAddress string `config:"doppler_address"` UaaAddress string `config:"uaa_address"` RlpAddress string `config:"rlp_address"` // ShardID when retrieving events from loggregator, sharing this ID across // multiple filebeats will shard the load of receiving and sending events. - ShardID string `config:"shard_id"` + ShardID string `config:"shard_id" validate:"required"` // Maximum amount of time to cache application objects from CF client. CacheDuration time.Duration `config:"cache_duration"` @@ -50,13 +48,6 @@ type Config struct { // InitDefaults initialize the defaults for the configuration. func (c *Config) InitDefaults() { - // If not provided by the user; subscription ID should be a unique string to avoid clustering by default. - // Default to using a UUID4 string. - uuid, err := uuid.NewV4() - if err != nil { - panic(err) - } - c.ShardID = uuid.String() c.CacheDuration = 120 * time.Second c.CacheRetryDelay = 20 * time.Second c.Version = ConsumerVersionV1 diff --git a/x-pack/libbeat/common/cloudfoundry/config_test.go b/x-pack/libbeat/common/cloudfoundry/config_test.go index bce98a0b642..79ba7d47cc1 100644 --- a/x-pack/libbeat/common/cloudfoundry/config_test.go +++ b/x-pack/libbeat/common/cloudfoundry/config_test.go @@ -22,26 +22,48 @@ func TestValidation(t *testing.T) { var noId Config assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&noId)) var noSecret Config assert.Error(t, ucfg.MustNewFrom(common.MapStr{ - "client_id": "client_id", + "api_address": "https://api.dev.cfdev.sh", + "client_id": "client_id", + "shard_id": "beats-test-1", }).Unpack(&noSecret)) + var noAPI Config + assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "client_id": "client_id", + "client_secret": "client_secret", + "shard_id": "beats-test-1", + }).Unpack(&noAPI)) + + var noShardID Config + assert.Error(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", + "client_id": "client_id", + "client_secret": "client_secret", + }).Unpack(&noShardID)) + var valid Config assert.NoError(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_id": "client_id", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&valid)) } func TestInitDefaults(t *testing.T) { var cfCfg Config assert.NoError(t, ucfg.MustNewFrom(common.MapStr{ + "api_address": "https://api.dev.cfdev.sh", "client_id": "client_id", "client_secret": "client_secret", + "shard_id": "beats-test-1", }).Unpack(&cfCfg)) - assert.Len(t, cfCfg.ShardID, 36) + assert.Equal(t, ConsumerVersionV1, cfCfg.Version) } diff --git a/x-pack/libbeat/common/cloudfoundry/hub.go b/x-pack/libbeat/common/cloudfoundry/hub.go index 4bb7fce1eec..4cf9757c278 100644 --- a/x-pack/libbeat/common/cloudfoundry/hub.go +++ b/x-pack/libbeat/common/cloudfoundry/hub.go @@ -5,10 +5,8 @@ package cloudfoundry import ( - "fmt" "net/http" "strings" - "time" "github.com/cloudfoundry-community/go-cfclient" "github.com/pkg/errors" @@ -19,11 +17,20 @@ import ( // Client interface exposed by Hub.Client. type Client interface { // GetAppByGuid returns the application from cloudfoundry. - GetAppByGuid(guid string) (*cfclient.App, error) - // StartJanitor keeps the cache of applications clean. - StartJanitor(interval time.Duration) - // StopJanitor stops the running janitor. - StopJanitor() + GetAppByGuid(guid string) (*AppMeta, error) + + // Close releases resources associated with this client. + Close() error +} + +// AppMeta is the metadata associated with a cloudfoundry application +type AppMeta struct { + Guid string `json:"guid"` + Name string `json:"name"` + SpaceGuid string `json:"space_guid"` + SpaceName string `json:"space_name"` + OrgGuid string `json:"org_guid"` + OrgName string `json:"org_name"` } // Hub is central place to get all the required clients to communicate with cloudfoundry. @@ -39,7 +46,7 @@ func NewHub(cfg *Config, userAgent string, log *logp.Logger) *Hub { } // Client returns the cloudfoundry client. -func (h *Hub) Client() (Client, error) { +func (h *Hub) Client() (*cfclient.Client, error) { httpClient, insecure, err := h.httpClient() if err != nil { return nil, err @@ -67,7 +74,15 @@ func (h *Hub) Client() (Client, error) { if h.cfg.UaaAddress != "" { cf.Endpoint.AuthEndpoint = h.cfg.UaaAddress } - return newClientCacheWrap(cf, h.cfg.CacheDuration, h.cfg.CacheRetryDelay, h.log), nil + return cf, nil +} + +func (h *Hub) ClientWithCache() (Client, error) { + c, err := h.Client() + if err != nil { + return nil, err + } + return newClientCacheWrap(c, h.cfg.APIAddress, h.cfg.CacheDuration, h.cfg.CacheRetryDelay, h.log) } // RlpListener returns a listener client that calls the passed callback when the provided events are streamed through @@ -85,7 +100,7 @@ func (h *Hub) RlpListener(callbacks RlpListenerCallbacks) (*RlpListener, error) // // In the case that the cloudfoundry client was already needed by the code path, call this method // as not to create a intermediate client that will not be used. -func (h *Hub) RlpListenerFromClient(client Client, callbacks RlpListenerCallbacks) (*RlpListener, error) { +func (h *Hub) RlpListenerFromClient(client *cfclient.Client, callbacks RlpListenerCallbacks) (*RlpListener, error) { var rlpAddress string if h.cfg.RlpAddress != "" { rlpAddress = h.cfg.RlpAddress @@ -107,47 +122,29 @@ func (h *Hub) DopplerConsumer(callbacks DopplerCallbacks) (*DopplerConsumer, err return h.DopplerConsumerFromClient(client, callbacks) } -func (h *Hub) DopplerConsumerFromClient(client Client, callbacks DopplerCallbacks) (*DopplerConsumer, error) { +func (h *Hub) DopplerConsumerFromClient(client *cfclient.Client, callbacks DopplerCallbacks) (*DopplerConsumer, error) { dopplerAddress := h.cfg.DopplerAddress if dopplerAddress == "" { - endpoint, err := cfEndpoint(client) - if err != nil { - return nil, errors.Wrap(err, "getting endpoints from client") - } - dopplerAddress = endpoint.DopplerEndpoint + dopplerAddress = client.Endpoint.DopplerEndpoint } httpClient, _, err := h.httpClient() if err != nil { return nil, errors.Wrap(err, "getting http client") } - // TODO: Refactor Client so it is easier to access the cfclient - ccw, ok := client.(*clientCacheWrap) - if !ok { - return nil, fmt.Errorf("client without cache wrap") - } - cfc, ok := ccw.client.(*cfclient.Client) - if !ok { - return nil, fmt.Errorf("client is not a cloud foundry client") - } - - tr := TokenRefresherFromCfClient(cfc) + tr := TokenRefresherFromCfClient(client) return newDopplerConsumer(dopplerAddress, h.cfg.ShardID, h.log, httpClient, tr, callbacks) } // doerFromClient returns an auth token doer using uaa. -func (h *Hub) doerFromClient(client Client) (*authTokenDoer, error) { +func (h *Hub) doerFromClient(client *cfclient.Client) (*authTokenDoer, error) { httpClient, _, err := h.httpClient() if err != nil { return nil, err } url := h.cfg.UaaAddress if url == "" { - endpoint, err := cfEndpoint(client) - if err != nil { - return nil, errors.Wrap(err, "getting endpoints from client") - } - url = endpoint.AuthEndpoint + url = client.Endpoint.AuthEndpoint } return newAuthTokenDoer(url, h.cfg.ClientID, h.cfg.ClientSecret, httpClient, h.log), nil } @@ -165,18 +162,6 @@ func (h *Hub) httpClient() (*http.Client, bool, error) { return httpClient, tls.InsecureSkipVerify, nil } -func cfEndpoint(client Client) (cfclient.Endpoint, error) { - ccw, ok := client.(*clientCacheWrap) - if !ok { - return cfclient.Endpoint{}, fmt.Errorf("client without cache wrap") - } - cfc, ok := ccw.client.(*cfclient.Client) - if !ok { - return cfclient.Endpoint{}, fmt.Errorf("client is not a cloud foundry client") - } - return cfc.Endpoint, nil -} - // defaultTransport returns a new http.Transport for http.Client func defaultTransport() *http.Transport { defaultTransport := http.DefaultTransport.(*http.Transport) diff --git a/x-pack/libbeat/common/cloudfoundry/main_test.go b/x-pack/libbeat/common/cloudfoundry/main_test.go new file mode 100644 index 00000000000..ec352def825 --- /dev/null +++ b/x-pack/libbeat/common/cloudfoundry/main_test.go @@ -0,0 +1,29 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package cloudfoundry + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/elastic/beats/v7/libbeat/paths" +) + +func TestMain(m *testing.M) { + // Override global beats data dir to avoid creating directories in the working copy. + tmpdir, err := ioutil.TempDir("", "beats-data-dir") + if err != nil { + fmt.Printf("Failed to create temporal data directory: %v\n", err) + os.Exit(1) + } + paths.Paths.Data = tmpdir + + result := m.Run() + os.RemoveAll(tmpdir) + + os.Exit(result) +} diff --git a/x-pack/libbeat/common/cloudfoundry/test/config.go b/x-pack/libbeat/common/cloudfoundry/test/config.go index f7b9cd18ffb..45059fc559c 100644 --- a/x-pack/libbeat/common/cloudfoundry/test/config.go +++ b/x-pack/libbeat/common/cloudfoundry/test/config.go @@ -7,15 +7,23 @@ package test import ( "os" "testing" + + "github.com/gofrs/uuid" ) func GetConfigFromEnv(t *testing.T) map[string]interface{} { t.Helper() + shardID, err := uuid.NewV4() + if err != nil { + t.Fatalf("Unable to create a random shard ID: %v", err) + } + config := map[string]interface{}{ "api_address": lookupEnv(t, "CLOUDFOUNDRY_API_ADDRESS"), "client_id": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_ID"), "client_secret": lookupEnv(t, "CLOUDFOUNDRY_CLIENT_SECRET"), + "shard_id": shardID.String(), "ssl.verification_mode": "none", } @@ -23,7 +31,6 @@ func GetConfigFromEnv(t *testing.T) map[string]interface{} { optionalConfig(config, "uaa_address", "CLOUDFOUNDRY_UAA_ADDRESS") optionalConfig(config, "rlp_address", "CLOUDFOUNDRY_RLP_ADDRESS") optionalConfig(config, "doppler_address", "CLOUDFOUNDRY_DOPPLER_ADDRESS") - optionalConfig(config, "shard_id", "CLOUDFOUNDRY_SHARD_ID") if t.Failed() { t.FailNow() diff --git a/x-pack/libbeat/persistentcache/encoding.go b/x-pack/libbeat/persistentcache/encoding.go new file mode 100644 index 00000000000..60b2058ebf4 --- /dev/null +++ b/x-pack/libbeat/persistentcache/encoding.go @@ -0,0 +1,55 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package persistentcache + +import ( + "bytes" + "encoding/json" + + ugorjicodec "github.com/ugorji/go/codec" +) + +type codec interface { + Decode([]byte, interface{}) error + Encode(interface{}) ([]byte, error) +} + +type jsonCodec struct{} + +func newJSONCodec() *jsonCodec { + return &jsonCodec{} +} + +// Encode encodes an object in json format. +func (*jsonCodec) Encode(v interface{}) ([]byte, error) { + return json.Marshal(v) +} + +// Decode decodes an object from its json representation. +func (*jsonCodec) Decode(d []byte, v interface{}) error { + return json.Unmarshal(d, v) +} + +type cborCodec struct { + handle ugorjicodec.CborHandle +} + +func newCBORCodec() *cborCodec { + return &cborCodec{} +} + +// Encode encodes an object in cbor format. +func (c *cborCodec) Encode(v interface{}) ([]byte, error) { + var buf bytes.Buffer + enc := ugorjicodec.NewEncoder(&buf, &c.handle) + err := enc.Encode(v) + return buf.Bytes(), err +} + +// Decode decodes an object from its cbor representation. +func (c *cborCodec) Decode(d []byte, v interface{}) error { + dec := ugorjicodec.NewDecoder(bytes.NewReader(d), &c.handle) + return dec.Decode(v) +} diff --git a/x-pack/libbeat/persistentcache/persistentcache.go b/x-pack/libbeat/persistentcache/persistentcache.go new file mode 100644 index 00000000000..39721095b86 --- /dev/null +++ b/x-pack/libbeat/persistentcache/persistentcache.go @@ -0,0 +1,112 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package persistentcache + +import ( + "fmt" + "time" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/libbeat/paths" +) + +const ( + cacheFile = "cache" + gcPeriod = 5 * time.Minute +) + +// PersistentCache is a persistent map of keys to values. Elements added to the +// cache are stored until they are explicitly deleted or are expired due to time-based +// eviction based on last access or add time. +type PersistentCache struct { + log *logp.Logger + + store *Store + codec codec + + refreshOnAccess bool + timeout time.Duration +} + +// Options are the options that can be used to customize persistent caches +type Options struct { + // Length of time before cache elements expire + Timeout time.Duration + + // If set to true, expiration time of an entry is updated + // when the object is accessed. + RefreshOnAccess bool + + // If empty, beats data path is used. + RootPath string +} + +// New creates and returns a new persistent cache. +// Cache returned by this method must be closed with Close() when +// not needed anymore. +func New(name string, opts Options) (*PersistentCache, error) { + logger := logp.NewLogger("persistentcache") + + rootPath := opts.RootPath + if rootPath == "" { + rootPath = paths.Resolve(paths.Data, cacheFile) + } + store, err := newStore(logger, rootPath, name) + if err != nil { + return nil, err + } + + return &PersistentCache{ + log: logger, + store: store, + codec: newCBORCodec(), + + refreshOnAccess: opts.RefreshOnAccess, + timeout: opts.Timeout, + }, nil +} + +// Put writes the given key and value to the map replacing any +// existing value if it exists. +func (c *PersistentCache) Put(k string, v interface{}) error { + return c.PutWithTimeout(k, v, 0) +} + +// PutWithTimeout writes the given key and value to the map replacing any +// existing value if it exists. +// The cache expiration time will be overwritten by timeout of the key being +// inserted. +func (c *PersistentCache) PutWithTimeout(k string, v interface{}, timeout time.Duration) error { + d, err := c.codec.Encode(v) + if err != nil { + return fmt.Errorf("encoding item to store in cache: %w", err) + } + if timeout == 0 { + timeout = c.timeout + } + return c.store.Set([]byte(k), d, timeout) +} + +// Get the current value associated with a key or nil if the key is not +// present. The last access time of the element is updated. +func (c *PersistentCache) Get(k string, v interface{}) error { + d, err := c.store.Get([]byte(k)) + if err != nil { + return err + } + if c.refreshOnAccess && c.timeout > 0 { + c.store.Set([]byte(k), d, c.timeout) + } + err = c.codec.Decode(d, v) + if err != nil { + return fmt.Errorf("decoding item stored in cache: %w", err) + } + return nil +} + +// Close releases all resources associated with this cache. +func (c *PersistentCache) Close() error { + return c.store.Close() +} diff --git a/x-pack/libbeat/persistentcache/persistentcache_test.go b/x-pack/libbeat/persistentcache/persistentcache_test.go new file mode 100644 index 00000000000..f6f28d73b75 --- /dev/null +++ b/x-pack/libbeat/persistentcache/persistentcache_test.go @@ -0,0 +1,436 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package persistentcache + +import ( + "fmt" + "io/ioutil" + "math/rand" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gofrs/uuid" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestPutGet(t *testing.T) { + logp.TestingSetup() + t.Parallel() + + cache, err := New("test", testOptions(t)) + require.NoError(t, err) + defer cache.Close() + + type valueType struct { + Something string + } + + var key = "somekey" + var value = valueType{Something: "foo"} + + err = cache.Put(key, value) + assert.NoError(t, err) + + var result valueType + err = cache.Get(key, &result) + assert.NoError(t, err) + assert.Equal(t, value, result) + + err = cache.Get("notexist", &result) + assert.Error(t, err) +} + +func TestPersist(t *testing.T) { + logp.TestingSetup() + t.Parallel() + + options := testOptions(t) + + cache, err := New("test", options) + require.NoError(t, err) + + type valueType struct { + Something string + } + + var key = "somekey" + var value = valueType{Something: "foo"} + + err = cache.Put(key, value) + assert.NoError(t, err) + + err = cache.Close() + assert.NoError(t, err) + + cache, err = New("test", options) + require.NoError(t, err) + defer cache.Close() + + var result valueType + err = cache.Get(key, &result) + assert.NoError(t, err) + assert.Equal(t, value, result) +} + +func TestExpired(t *testing.T) { + if testing.Short() { + t.Skip("skipping in short mode") + } + logp.TestingSetup() + t.Parallel() + + options := testOptions(t) + cache, err := New("test", options) + require.NoError(t, err) + defer cache.Close() + + type valueType struct { + Something string + } + + var key = "somekey" + var value = valueType{Something: "foo"} + + // Badger TTL is not reliable on sub-second durations. + err = cache.PutWithTimeout(key, value, 2*time.Second) + assert.NoError(t, err) + + var result valueType + err = cache.Get(key, &result) + assert.NoError(t, err) + assert.Equal(t, value, result) + + time.Sleep(2 * time.Second) + err = cache.Get(key, &result) + assert.Error(t, err) +} + +func TestRefreshOnAccess(t *testing.T) { + t.Skip("flaky test") + + if testing.Short() { + t.Skip("skipping in short mode") + } + + logp.TestingSetup() + t.Parallel() + + // Badger TTL is not reliable on sub-second durations. + options := testOptions(t) + options.Timeout = 2 * time.Second + options.RefreshOnAccess = true + + cache, err := New("test", options) + require.NoError(t, err) + defer cache.Close() + + type valueType struct { + Something string + } + + var key1 = "somekey" + var value1 = valueType{Something: "foo"} + var key2 = "otherkey" + var value2 = valueType{Something: "bar"} + + err = cache.Put(key1, value1) + assert.NoError(t, err) + err = cache.Put(key2, value2) + assert.NoError(t, err) + + time.Sleep(1 * time.Second) + + var result valueType + err = cache.Get(key1, &result) + assert.NoError(t, err) + assert.Equal(t, value1, result) + + time.Sleep(1 * time.Second) + + err = cache.Get(key1, &result) + assert.NoError(t, err) + assert.Equal(t, value1, result) + err = cache.Get(key2, &result) + assert.Error(t, err) +} + +var benchmarkCacheSizes = []int{10, 100, 1000, 10000, 100000} + +func BenchmarkPut(b *testing.B) { + type cache interface { + Put(key string, value interface{}) error + Close() error + } + + options := testOptions(b) + newPersistentCache := func(tb testing.TB, name string) cache { + cache, err := New(name, options) + require.NoError(tb, err) + return cache + } + + caches := []struct { + name string + factory func(t testing.TB, name string) cache + }{ + {name: "badger", factory: newPersistentCache}, + } + + b.Run("random strings", func(b *testing.B) { + for _, c := range caches { + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + cache := c.factory(b, b.Name()) + defer cache.Close() + + value := uuid.Must(uuid.NewV4()).String() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := cache.Put(strconv.Itoa(i), value) + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + }) + } + }) + + b.Run("objects", func(b *testing.B) { + for _, c := range caches { + type entry struct { + ID string + Data [128]byte + } + + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + cache := c.factory(b, b.Name()) + defer cache.Close() + + value := entry{ID: uuid.Must(uuid.NewV4()).String()} + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := cache.Put(strconv.Itoa(i), value) + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + }) + } + }) + + b.Run("maps", func(b *testing.B) { + for _, c := range caches { + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + cache := c.factory(b, b.Name()) + defer cache.Close() + + value := map[string]string{ + "id": uuid.Must(uuid.NewV4()).String(), + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := cache.Put(strconv.Itoa(i), value) + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + }) + } + }) + + for _, size := range benchmarkCacheSizes { + b.Run(fmt.Sprintf("%d objects", size), func(b *testing.B) { + type entry struct { + ID string + Data [128]byte + } + objects := make([]entry, size) + for i := 0; i < size; i++ { + objects[i] = entry{ + ID: uuid.Must(uuid.NewV4()).String(), + } + } + + for _, c := range caches { + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + cache := c.factory(b, b.Name()) + for _, object := range objects { + cache.Put(object.ID, object) + } + cache.Close() + } + }) + } + }) + } +} + +func BenchmarkOpen(b *testing.B) { + type cache interface { + Put(key string, value interface{}) error + Close() error + } + + options := testOptions(b) + newPersistentCache := func(tb testing.TB, name string) cache { + cache, err := New(name, options) + require.NoError(tb, err) + return cache + } + + caches := []struct { + name string + factory func(t testing.TB, name string) cache + }{ + {name: "badger", factory: newPersistentCache}, + } + + for _, size := range benchmarkCacheSizes { + b.Run(fmt.Sprintf("%d objects", size), func(b *testing.B) { + type entry struct { + ID string + Data [128]byte + } + + for _, c := range caches { + cacheName := b.Name() + cache := c.factory(b, cacheName) + for i := 0; i < size; i++ { + e := entry{ + ID: uuid.Must(uuid.NewV4()).String(), + } + err := cache.Put(e.ID, e) + require.NoError(b, err) + } + cache.Close() + + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + cache := c.factory(b, cacheName) + cache.Close() + } + }) + } + }) + } +} + +func BenchmarkGet(b *testing.B) { + type cache interface { + Put(key string, value interface{}) error + Get(key string, value interface{}) error + Close() error + } + + options := testOptions(b) + newPersistentCache := func(tb testing.TB, name string) cache { + cache, err := New(name, options) + require.NoError(tb, err) + return cache + } + + caches := []struct { + name string + factory func(t testing.TB, name string) cache + }{ + {name: "badger", factory: newPersistentCache}, + } + + for _, size := range benchmarkCacheSizes { + b.Run(fmt.Sprintf("%d objects", size), func(b *testing.B) { + for _, c := range caches { + type entry struct { + ID string + Data [128]byte + } + + cacheName := b.Name() + + objects := make([]entry, size) + cache := c.factory(b, cacheName) + for i := 0; i < size; i++ { + e := entry{ + ID: uuid.Must(uuid.NewV4()).String(), + } + objects[i] = e + err := cache.Put(e.ID, e) + require.NoError(b, err) + } + cache.Close() + + b.Run(c.name, func(b *testing.B) { + b.ReportAllocs() + + cache := c.factory(b, cacheName) + + var result entry + + b.ResetTimer() + for i := 0; i < b.N; i++ { + expected := objects[rand.Intn(size)] + cache.Get(expected.ID, &result) + if expected.ID != result.ID { + b.Fatalf("%s != %s", expected.ID, result.ID) + } + } + b.StopTimer() + cache.Close() + }) + } + }) + } +} + +func testOptions(t testing.TB) Options { + t.Helper() + + tempDir, err := ioutil.TempDir("", "beat-data-dir-") + require.NoError(t, err) + + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + return Options{ + RootPath: filepath.Join(tempDir, cacheFile), + } +} + +func dirSize(tb testing.TB, path string) int64 { + var size int64 + + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + require.NoError(tb, err) + + return size +} diff --git a/x-pack/libbeat/persistentcache/store.go b/x-pack/libbeat/persistentcache/store.go new file mode 100644 index 00000000000..e14b90fedda --- /dev/null +++ b/x-pack/libbeat/persistentcache/store.go @@ -0,0 +1,138 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package persistentcache + +import ( + "fmt" + "os" + "path/filepath" + "time" + + badger "github.com/dgraph-io/badger/v2" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +// Store is a store for a persistent cache. It can be shared between consumers. +type Store struct { + logger *logp.Logger + name string + + gcQuit chan struct{} + db *badger.DB +} + +// newStore opens a store persisted on the specified directory, with the specified name. +// If succeeds, returned store must be closed. +func newStore(logger *logp.Logger, dir, name string) (*Store, error) { + dbPath := filepath.Join(dir, name) + err := os.MkdirAll(dbPath, 0750) + if err != nil { + return nil, fmt.Errorf("creating directory for cache store: %w", err) + } + + // Opinionated options for the use of badger as a store for metadata caches in Beats. + options := badger.DefaultOptions(dbPath) + options.Logger = badgerLogger{logger.Named("badger")} + options.SyncWrites = false + + db, err := badger.Open(options) + if err != nil { + return nil, fmt.Errorf("opening database for cache store: %w", err) + } + + store := Store{ + db: db, + logger: logger, + name: name, + gcQuit: make(chan struct{}), + } + go store.runGC(gcPeriod) + return &store, nil +} + +// Close closes the store. +func (s *Store) Close() error { + s.stopGC() + err := s.db.Close() + if err != nil { + return fmt.Errorf("closing database of cache store: %w", err) + } + return nil +} + +// Set sets a value with a ttl in the store. If ttl is zero, it is ignored. +func (s *Store) Set(k, v []byte, ttl time.Duration) error { + entry := badger.Entry{ + Key: []byte(k), + Value: v, + } + if ttl > 0 { + entry.WithTTL(ttl) + } + err := s.db.Update(func(txn *badger.Txn) error { + return txn.SetEntry(&entry) + }) + if err != nil { + return fmt.Errorf("setting value in cache store: %w", err) + } + return err +} + +// Get gets a value from the store. +func (s *Store) Get(k []byte) ([]byte, error) { + var result []byte + err := s.db.View(func(txn *badger.Txn) error { + item, err := txn.Get(k) + if err != nil { + return err + } + result, err = item.ValueCopy(nil) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("getting value from cache store: %w", err) + } + return result, nil +} + +// runGC starts garbage collection in the store. +func (s *Store) runGC(period time.Duration) { + ticker := time.NewTicker(period) + defer ticker.Stop() + for { + select { + case <-ticker.C: + var err error + count := 0 + for err == nil { + err = s.db.RunValueLogGC(0.5) + count++ + } + s.logger.Debugf("Result of garbage collector after running %d times: %s", count, err) + case <-s.gcQuit: + return + } + } + +} + +// stopGC stops garbage collection in the store. +func (s *Store) stopGC() { + close(s.gcQuit) +} + +// badgerLogger is an adapter between a logp logger and the loggers expected by badger. +type badgerLogger struct { + *logp.Logger +} + +// Warningf logs a message at the warning level. +func (l badgerLogger) Warningf(format string, args ...interface{}) { + l.Warnf(format, args...) +} diff --git a/x-pack/libbeat/persistentcache/store_test.go b/x-pack/libbeat/persistentcache/store_test.go new file mode 100644 index 00000000000..efa51f24b7f --- /dev/null +++ b/x-pack/libbeat/persistentcache/store_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package persistentcache + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/beats/v7/libbeat/logp" +) + +func TestStandaloneStore(t *testing.T) { + type valueType struct { + Something string + } + + var key = []byte("somekey") + var value = []byte("somevalue") + + tempDir, err := ioutil.TempDir("", "beat-data-dir-") + require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(tempDir) }) + + store, err := newStore(logp.NewLogger("test"), tempDir, "store-cache") + require.NoError(t, err) + + err = store.Set(key, value, 0) + assert.NoError(t, err) + + result, err := store.Get(key) + if assert.NoError(t, err) { + assert.Equal(t, value, result) + } + + err = store.Close() + assert.NoError(t, err) +} diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go index d18a04ca979..a6b8bd16566 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata.go @@ -5,8 +5,7 @@ package add_cloudfoundry_metadata import ( - "time" - + "github.com/gofrs/uuid" "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/beat" @@ -34,20 +33,22 @@ const selector = "add_cloudfoundry_metadata" // New constructs a new add_cloudfoundry_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { var config cloudfoundry.Config + + // ShardID is required in cloudfoundry config to consume from the firehose, + // but not for metadata requests, randomly generate one and use it. + config.ShardID = uuid.Must(uuid.NewV4()).String() + if err := cfg.Unpack(&config); err != nil { return nil, errors.Wrapf(err, "fail to unpack the %v configuration", processorName) } log := logp.NewLogger(selector) hub := cloudfoundry.NewHub(&config, "add_cloudfoundry_metadata", log) - client, err := hub.Client() + client, err := hub.ClientWithCache() if err != nil { - log.Debugf("%s: failed to created cloudfoundry client: %+v", processorName, err) + return nil, errors.Wrapf(err, "%s: creating cloudfoundry client", processorName) } - // Janitor run every 5 minutes to clean up the client cache. - client.StartJanitor(5 * time.Minute) - return &addCloudFoundryMetadata{ log: log, client: client, @@ -79,18 +80,31 @@ func (d *addCloudFoundryMetadata) Run(event *beat.Event) (*beat.Event, error) { "name": app.Name, }, "space": common.MapStr{ - "id": app.SpaceData.Meta.Guid, - "name": app.SpaceData.Entity.Name, + "id": app.SpaceGuid, + "name": app.SpaceName, }, "org": common.MapStr{ - "id": app.SpaceData.Entity.OrgData.Meta.Guid, - "name": app.SpaceData.Entity.OrgData.Entity.Name, + "id": app.OrgGuid, + "name": app.OrgName, }, }, }) return event, nil } +// String returns this processor name. func (d *addCloudFoundryMetadata) String() string { return processorName } + +// Close closes the underlying client and releases its resources. +func (d *addCloudFoundryMetadata) Close() error { + if d.client == nil { + return nil + } + err := d.client.Close() + if err != nil { + return errors.Wrap(err, "closing client") + } + return nil +} diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go index 1aff4cb2df8..95a7073321e 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/add_cloudfoundry_metadata_test.go @@ -6,7 +6,6 @@ package add_cloudfoundry_metadata import ( "testing" - "time" "github.com/cloudfoundry-community/go-cfclient" "github.com/gofrs/uuid" @@ -15,6 +14,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/x-pack/libbeat/common/cloudfoundry" ) func TestNoClient(t *testing.T) { @@ -84,25 +84,13 @@ func TestCFAppNotFound(t *testing.T) { func TestCFAppUpdated(t *testing.T) { guid := mustCreateFakeGuid() - app := cfclient.App{ - Guid: guid, - Name: "My Fake App", - SpaceData: cfclient.SpaceResource{ - Meta: cfclient.Meta{ - Guid: mustCreateFakeGuid(), - }, - Entity: cfclient.Space{ - Name: "My Fake Space", - OrgData: cfclient.OrgResource{ - Meta: cfclient.Meta{ - Guid: mustCreateFakeGuid(), - }, - Entity: cfclient.Org{ - Name: "My Fake Org", - }, - }, - }, - }, + app := cloudfoundry.AppMeta{ + Guid: guid, + Name: "My Fake App", + SpaceGuid: mustCreateFakeGuid(), + SpaceName: "My Fake Space", + OrgGuid: mustCreateFakeGuid(), + OrgName: "My Fake Org", } p := addCloudFoundryMetadata{ log: logp.NewLogger("add_cloudfoundry_metadata"), @@ -126,12 +114,12 @@ func TestCFAppUpdated(t *testing.T) { "name": app.Name, }, "space": common.MapStr{ - "id": app.SpaceData.Meta.Guid, - "name": app.SpaceData.Entity.Name, + "id": app.SpaceGuid, + "name": app.SpaceName, }, "org": common.MapStr{ - "id": app.SpaceData.Entity.OrgData.Meta.Guid, - "name": app.SpaceData.Entity.OrgData.Entity.Name, + "id": app.OrgGuid, + "name": app.OrgName, }, }, }, @@ -142,20 +130,18 @@ func TestCFAppUpdated(t *testing.T) { } type fakeClient struct { - app cfclient.App + app cloudfoundry.AppMeta } -func (c *fakeClient) GetAppByGuid(guid string) (*cfclient.App, error) { +func (c *fakeClient) GetAppByGuid(guid string) (*cloudfoundry.AppMeta, error) { if c.app.Guid != guid { return nil, cfclient.CloudFoundryError{Code: 100004} } return &c.app, nil } -func (c *fakeClient) StartJanitor(_ time.Duration) { -} - -func (c *fakeClient) StopJanitor() { +func (c *fakeClient) Close() error { + return nil } func mustCreateFakeGuid() string { diff --git a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc index 558b5a1031b..67e89c8173b 100644 --- a/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_cloudfoundry_metadata/docs/add_cloudfoundry_metadata.asciidoc @@ -6,8 +6,6 @@ add_cloudfoundry_metadata ++++ -beta[] - The `add_cloudfoundry_metadata` processor annotates each event with relevant metadata from Cloud Foundry applications. The events are annotated with Cloud Foundry metadata, only if the event contains a reference to a Cloud Foundry application (using field diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 00fa2c1265e..d02dc5e16aa 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -35,3 +35,14 @@ stages: mage: "mage build unitTest" platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/metricbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index ff9bffda33e..41552410a38 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -343,6 +343,14 @@ metricbeat.modules: metrics: - id: ["requests/count", "requests/duration"] +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' + #--------------------------------- Beat Module --------------------------------- - module: beat metricsets: @@ -390,6 +398,7 @@ metricbeat.modules: rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 #----------------------------- CockroachDB Module ----------------------------- @@ -685,6 +694,13 @@ metricbeat.modules: # use istio-pilot.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset hosts: ["localhost:15014"] +# Istio istiod to monitor the Istio Daemon for versions after 1.5 of Istio. +- module: istio + metricsets: ['istiod'] + period: 10s + # use istiod.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset + hosts: ['localhost:15014'] + #------------------------------- Jolokia Module ------------------------------- - module: jolokia #metricsets: ["jmx"] @@ -894,8 +910,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/x-pack/metricbeat/module/aws/_meta/config.yml b/x-pack/metricbeat/module/aws/_meta/config.yml index 618ed4cd854..6f604138505 100644 --- a/x-pack/metricbeat/module/aws/_meta/config.yml +++ b/x-pack/metricbeat/module/aws/_meta/config.yml @@ -44,4 +44,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request diff --git a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc index fe9aeea007f..e4e55e82136 100644 --- a/x-pack/metricbeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/aws/_meta/docs.asciidoc @@ -11,16 +11,27 @@ module. Please see <> for more details. [float] == Module-specific configuration notes +* *AWS Credentials* + The `aws` module requires AWS credentials configuration in order to make AWS API calls. Users can either use `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and/or `AWS_SESSION_TOKEN`, or use shared AWS credentials file. Please see <> for more details. +* *regions* + This module also accepts optional configuration `regions` to specify which AWS regions to query metrics from. If the `regions` parameter is not set in the config file, then by default, the `aws` module will query metrics from all available AWS regions. +* *latency* + +Some AWS services send monitoring metrics to CloudWatch with a latency to +process larger than Metricbeat collection period. This case, please specify a +`latency` parameter so collection start time and end time will be shifted by the +given latency amount. + The aws module comes with a predefined dashboard. For example: image::./images/metricbeat-aws-overview.png[] diff --git a/x-pack/metricbeat/module/aws/aws.go b/x-pack/metricbeat/module/aws/aws.go index f7b744c27cb..167e6a088a0 100644 --- a/x-pack/metricbeat/module/aws/aws.go +++ b/x-pack/metricbeat/module/aws/aws.go @@ -27,6 +27,7 @@ import ( type Config struct { Period time.Duration `config:"period" validate:"nonzero,required"` Regions []string `config:"regions"` + Latency time.Duration `config:"latency"` AWSConfig awscommon.ConfigAWS `config:",inline"` TagsFilter []Tag `config:"tags_filter"` } @@ -37,6 +38,7 @@ type MetricSet struct { RegionsList []string Endpoint string Period time.Duration + Latency time.Duration AwsConfig *awssdk.Config AccountName string AccountID string @@ -87,6 +89,7 @@ func NewMetricSet(base mb.BaseMetricSet) (*MetricSet, error) { metricSet := MetricSet{ BaseMetricSet: base, Period: config.Period, + Latency: config.Latency, AwsConfig: &awsConfig, TagsFilter: config.TagsFilter, } @@ -193,11 +196,14 @@ func StringInSlice(str string, list []string) (bool, int) { } // InitEvent initialize mb.Event with basic information like service.name, cloud.provider -func InitEvent(regionName string, accountName string, accountID string) mb.Event { - event := mb.Event{} - event.MetricSetFields = common.MapStr{} - event.ModuleFields = common.MapStr{} - event.RootFields = common.MapStr{} +func InitEvent(regionName string, accountName string, accountID string, timestamp time.Time) mb.Event { + event := mb.Event{ + Timestamp: timestamp, + MetricSetFields: common.MapStr{}, + ModuleFields: common.MapStr{}, + RootFields: common.MapStr{}, + } + event.RootFields.Put("cloud.provider", "aws") if regionName != "" { event.RootFields.Put("cloud.region", regionName) diff --git a/x-pack/metricbeat/module/aws/billing/billing.go b/x-pack/metricbeat/module/aws/billing/billing.go index 2eb2bd2854a..39d600983d0 100644 --- a/x-pack/metricbeat/module/aws/billing/billing.go +++ b/x-pack/metricbeat/module/aws/billing/billing.go @@ -116,7 +116,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { startDate, endDate := getStartDateEndDate(m.Period) // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) // get cost metrics from cost explorer awsConfig := m.MetricSet.AwsConfig.Copy() @@ -178,26 +178,28 @@ func (m *MetricSet) getCloudWatchBillingMetrics( // Find a timestamp for all metrics in output timestamp := aws.FindTimestamp(metricDataOutput) - if !timestamp.IsZero() { - for _, output := range metricDataOutput { - if len(output.Values) == 0 { - continue - } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, labelSeparator) + if timestamp.IsZero() { + return nil + } + + for _, output := range metricDataOutput { + if len(output.Values) == 0 { + continue + } + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, labelSeparator) - event := aws.InitEvent("", m.AccountName, m.AccountID) - event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) + event := aws.InitEvent("", m.AccountName, m.AccountID, timestamp) + event.MetricSetFields.Put(labels[0], output.Values[timestampIdx]) - i := 1 - for i < len(labels)-1 { - event.MetricSetFields.Put(labels[i], labels[i+1]) - i += 2 - } - event.Timestamp = endTime - events = append(events, event) + i := 1 + for i < len(labels)-1 { + event.MetricSetFields.Put(labels[i], labels[i+1]) + i += 2 } + event.Timestamp = endTime + events = append(events, event) } } return events @@ -278,7 +280,7 @@ func (m *MetricSet) getCostGroupBy(svcCostExplorer costexploreriface.ClientAPI, } func (m *MetricSet) addCostMetrics(metrics map[string]costexplorer.MetricValue, groupDefinition costexplorer.GroupDefinition, startDate string, endDate string) mb.Event { - event := aws.InitEvent("", m.AccountName, m.AccountID) + event := aws.InitEvent("", m.AccountName, m.AccountID, time.Now()) // add group definition event.MetricSetFields.Put("group_definition", common.MapStr{ diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go index 42d68acb3af..af04508a111 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch.go @@ -141,7 +141,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // Check statistic method in config err := m.checkStatistics() @@ -498,34 +499,35 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes // Find a timestamp for all metrics in output timestamp := aws.FindTimestamp(metricDataResults) + if timestamp.IsZero() { + return nil, nil + } // Create events when there is no tags_filter or resource_type specified. if len(resourceTypeTagFilters) == 0 { - if !timestamp.IsZero() { - for _, output := range metricDataResults { - if len(output.Values) == 0 { - continue - } + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, labelSeparator) - if len(labels) != 5 { - // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] - if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID) - } - events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) - continue + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, labelSeparator) + if len(labels) != 5 { + // when there is no identifier value in label, use region+accountID+namespace instead + identifier := regionName + m.AccountID + labels[namespaceIdx] + if _, ok := events[identifier]; !ok { + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) } + events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) + continue + } - identifierValue := labels[identifierValueIdx] - if _, ok := events[identifierValue]; !ok { - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID) - } - events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + identifierValue := labels[identifierValueIdx] + if _, ok := events[identifierValue]; !ok { + events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) } + events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) } } return events, nil @@ -555,45 +557,43 @@ func (m *MetricSet) createEvents(svcCloudwatch cloudwatchiface.ClientAPI, svcRes m.logger.Debugf("In region %s, service %s tags match tags_filter", regionName, identifier) } - if !timestamp.IsZero() { - for _, output := range metricDataResults { - if len(output.Values) == 0 { - continue - } + for _, output := range metricDataResults { + if len(output.Values) == 0 { + continue + } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, labelSeparator) - if len(labels) != 5 { - // if there is no tag in labels but there is a tagsFilter, then no event should be reported. - if len(tagsFilter) != 0 { - continue - } - - // when there is no identifier value in label, use region+accountID+namespace instead - identifier := regionName + m.AccountID + labels[namespaceIdx] - if _, ok := events[identifier]; !ok { - events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID) - } - events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, labelSeparator) + if len(labels) != 5 { + // if there is no tag in labels but there is a tagsFilter, then no event should be reported. + if len(tagsFilter) != 0 { continue } - identifierValue := labels[identifierValueIdx] - if _, ok := events[identifierValue]; !ok { - // when tagsFilter is not empty but no entry in - // resourceTagMap for this identifier, do not initialize - // an event for this identifier. - if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { - continue - } - events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID) + // when there is no identifier value in label, use region+accountID+namespace instead + identifier := regionName + m.AccountID + labels[namespaceIdx] + if _, ok := events[identifier]; !ok { + events[identifier] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) } - events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + events[identifier] = insertRootFields(events[identifier], output.Values[timestampIdx], labels) + continue + } - // add tags to event based on identifierValue - insertTags(events, identifierValue, resourceTagMap) + identifierValue := labels[identifierValueIdx] + if _, ok := events[identifierValue]; !ok { + // when tagsFilter is not empty but no entry in + // resourceTagMap for this identifier, do not initialize + // an event for this identifier. + if len(tagsFilter) != 0 && resourceTagMap[identifierValue] == nil { + continue + } + events[identifierValue] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) } + events[identifierValue] = insertRootFields(events[identifierValue], output.Values[timestampIdx], labels) + + // add tags to event based on identifierValue + insertTags(events, identifierValue, resourceTagMap) } } } diff --git a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go index 353ffd0e236..ecd4bb2f9d1 100644 --- a/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go +++ b/x-pack/metricbeat/module/aws/cloudwatch/cloudwatch_test.go @@ -11,9 +11,6 @@ import ( "testing" "time" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/metricbeat/mb" - awssdk "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/cloudwatch" "github.com/aws/aws-sdk-go-v2/service/cloudwatch/cloudwatchiface" @@ -22,12 +19,14 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/assert" + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/aws" ) var ( regionName = "us-west-1" - timestamp = time.Now() + timestamp = time.Date(2020, 10, 06, 00, 00, 00, 0, time.UTC) accountID = "123456789012" accountName = "test" @@ -1357,7 +1356,7 @@ func TestCreateEventsWithIdentifier(t *testing.T) { Value: "test-ec2", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1399,7 +1398,7 @@ func TestCreateEventsWithoutIdentifier(t *testing.T) { } resourceTypeTagFilters := map[string][]aws.Tag{} - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1447,7 +1446,7 @@ func TestCreateEventsWithTagsFilter(t *testing.T) { Value: "foo", }, } - startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) events, err := m.createEvents(mockCloudwatchSvc, mockTaggingSvc, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) assert.NoError(t, err) @@ -1466,9 +1465,9 @@ func TestInsertTags(t *testing.T) { tagValue3 := "dev" events := map[string]mb.Event{} - events[identifier1] = aws.InitEvent(regionName, accountName, accountID) - events[identifier2] = aws.InitEvent(regionName, accountName, accountID) - events[identifierContainsArn] = aws.InitEvent(regionName, accountName, accountID) + events[identifier1] = aws.InitEvent(regionName, accountName, accountID, timestamp) + events[identifier2] = aws.InitEvent(regionName, accountName, accountID, timestamp) + events[identifierContainsArn] = aws.InitEvent(regionName, accountName, accountID, timestamp) resourceTagMap := map[string][]resourcegroupstaggingapi.Tag{} resourceTagMap["test-s3-1"] = []resourcegroupstaggingapi.Tag{ @@ -1569,3 +1568,29 @@ func TestConfigDimensionValueContainsWildcard(t *testing.T) { }) } } + +func TestCreateEventsTimestamp(t *testing.T) { + m := MetricSet{ + logger: logp.NewLogger("test"), + CloudwatchConfigs: []Config{{Statistic: []string{"Average"}}}, + MetricSet: &aws.MetricSet{Period: 5, AccountID: accountID}, + } + + listMetricWithStatsTotal := []metricsWithStatistics{ + { + cloudwatch.Metric{ + MetricName: awssdk.String("CPUUtilization"), + Namespace: awssdk.String("AWS/EC2"), + }, + []string{"Average"}, + nil, + }, + } + + resourceTypeTagFilters := map[string][]aws.Tag{} + startTime, endTime := aws.GetStartTimeEndTime(m.MetricSet.Period, m.MetricSet.Latency) + + events, err := m.createEvents(&MockCloudWatchClientWithoutDim{}, &MockResourceGroupsTaggingClient{}, listMetricWithStatsTotal, resourceTypeTagFilters, regionName, startTime, endTime) + assert.NoError(t, err) + assert.Equal(t, timestamp, events[regionName+accountID+namespace].Timestamp) +} diff --git a/x-pack/metricbeat/module/aws/ec2/ec2.go b/x-pack/metricbeat/module/aws/ec2/ec2.go index 36ad9a1ca02..7cb69623a00 100644 --- a/x-pack/metricbeat/module/aws/ec2/ec2.go +++ b/x-pack/metricbeat/module/aws/ec2/ec2.go @@ -88,7 +88,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() @@ -174,115 +175,117 @@ func constructMetricQueries(listMetricsOutput []cloudwatch.Metric, instanceID st } func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.MetricDataResult, instanceOutput map[string]ec2.Instance, regionName string) (map[string]mb.Event, error) { + // monitoring state for each instance + monitoringStates := map[string]string{} + + // Find a timestamp for all metrics in output + timestamp := aws.FindTimestamp(getMetricDataResults) + if timestamp.IsZero() { + return nil, nil + } + // Initialize events and metricSetFieldResults per instanceID events := map[string]mb.Event{} metricSetFieldResults := map[idStat]map[string]interface{}{} for instanceID := range instanceOutput { for _, statistic := range statistics { - events[instanceID] = aws.InitEvent(regionName, m.AccountName, m.AccountID) + events[instanceID] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) metricSetFieldResults[idStat{instanceID: instanceID, statistic: statistic}] = map[string]interface{}{} } } - // monitoring state for each instance - monitoringStates := map[string]string{} + for _, output := range getMetricDataResults { + if len(output.Values) == 0 { + continue + } - // Find a timestamp for all metrics in output - timestamp := aws.FindTimestamp(getMetricDataResults) - if !timestamp.IsZero() { - for _, output := range getMetricDataResults { - if len(output.Values) == 0 { + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + label, err := newLabelFromJSON(*output.Label) + if err != nil { + m.logger.Errorf("convert cloudwatch MetricDataResult label failed for label = %s: %w", *output.Label, err) continue } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - label, err := newLabelFromJSON(*output.Label) - if err != nil { - m.logger.Errorf("convert cloudwatch MetricDataResult label failed for label = %s: %w", *output.Label, err) + instanceID := label.InstanceID + statistic := label.Statistic + + // Add tags + tags := instanceOutput[instanceID].Tags + if m.TagsFilter != nil { + // Check with each tag filter + // If tag filter doesn't exist in tagKeys/tagValues, + // then do not report this event/instance. + if exists := aws.CheckTagFiltersExist(m.TagsFilter, tags); !exists { + // if tag filter doesn't exist, remove this event initial + // entry to avoid report an empty event. + delete(events, instanceID) continue } + } - instanceID := label.InstanceID - statistic := label.Statistic - - // Add tags - tags := instanceOutput[instanceID].Tags - if m.TagsFilter != nil { - // Check with each tag filter - // If tag filter doesn't exist in tagKeys/tagValues, - // then do not report this event/instance. - if exists := aws.CheckTagFiltersExist(m.TagsFilter, tags); !exists { - // if tag filter doesn't exist, remove this event initial - // entry to avoid report an empty event. - delete(events, instanceID) - continue - } - } - - // By default, replace dot "." using underscore "_" for tag keys. - // Note: tag values are not dedotted. - for _, tag := range tags { - events[instanceID].ModuleFields.Put("tags."+common.DeDot(*tag.Key), *tag.Value) - // add cloud.instance.name and host.name into ec2 events - if *tag.Key == "Name" { - events[instanceID].RootFields.Put("cloud.instance.name", *tag.Value) - events[instanceID].RootFields.Put("host.name", *tag.Value) - } - } - - machineType, err := instanceOutput[instanceID].InstanceType.MarshalValue() - if err != nil { - return events, errors.Wrap(err, "instance.InstanceType.MarshalValue failed") + // By default, replace dot "." using underscore "_" for tag keys. + // Note: tag values are not dedotted. + for _, tag := range tags { + events[instanceID].ModuleFields.Put("tags."+common.DeDot(*tag.Key), *tag.Value) + // add cloud.instance.name and host.name into ec2 events + if *tag.Key == "Name" { + events[instanceID].RootFields.Put("cloud.instance.name", *tag.Value) + events[instanceID].RootFields.Put("host.name", *tag.Value) } + } - events[instanceID].RootFields.Put("cloud.instance.id", instanceID) - events[instanceID].RootFields.Put("cloud.machine.type", machineType) + machineType, err := instanceOutput[instanceID].InstanceType.MarshalValue() + if err != nil { + return events, errors.Wrap(err, "instance.InstanceType.MarshalValue failed") + } - placement := instanceOutput[instanceID].Placement - if placement != nil { - events[instanceID].RootFields.Put("cloud.availability_zone", *placement.AvailabilityZone) - } + events[instanceID].RootFields.Put("cloud.instance.id", instanceID) + events[instanceID].RootFields.Put("cloud.machine.type", machineType) - if len(output.Values) > timestampIdx { - metricSetFieldResults[idStat{instanceID: instanceID, statistic: statistic}][label.MetricName] = fmt.Sprint(output.Values[timestampIdx]) - } + placement := instanceOutput[instanceID].Placement + if placement != nil { + events[instanceID].RootFields.Put("cloud.availability_zone", *placement.AvailabilityZone) + } - instanceStateName, err := instanceOutput[instanceID].State.Name.MarshalValue() - if err != nil { - return events, errors.Wrap(err, "instance.State.Name.MarshalValue failed") - } + if len(output.Values) > timestampIdx { + metricSetFieldResults[idStat{instanceID: instanceID, statistic: statistic}][label.MetricName] = fmt.Sprint(output.Values[timestampIdx]) + } - monitoringState, err := instanceOutput[instanceID].Monitoring.State.MarshalValue() - if err != nil { - return events, errors.Wrap(err, "instance.Monitoring.State.MarshalValue failed") - } + instanceStateName, err := instanceOutput[instanceID].State.Name.MarshalValue() + if err != nil { + return events, errors.Wrap(err, "instance.State.Name.MarshalValue failed") + } - monitoringStates[instanceID] = monitoringState + monitoringState, err := instanceOutput[instanceID].Monitoring.State.MarshalValue() + if err != nil { + return events, errors.Wrap(err, "instance.Monitoring.State.MarshalValue failed") + } - cpuOptions := instanceOutput[instanceID].CpuOptions - if cpuOptions != nil { - events[instanceID].MetricSetFields.Put("instance.core.count", *cpuOptions.CoreCount) - events[instanceID].MetricSetFields.Put("instance.threads_per_core", *cpuOptions.ThreadsPerCore) - } + monitoringStates[instanceID] = monitoringState - publicIP := instanceOutput[instanceID].PublicIpAddress - if publicIP != nil { - events[instanceID].MetricSetFields.Put("instance.public.ip", *publicIP) - } + cpuOptions := instanceOutput[instanceID].CpuOptions + if cpuOptions != nil { + events[instanceID].MetricSetFields.Put("instance.core.count", *cpuOptions.CoreCount) + events[instanceID].MetricSetFields.Put("instance.threads_per_core", *cpuOptions.ThreadsPerCore) + } - privateIP := instanceOutput[instanceID].PrivateIpAddress - if privateIP != nil { - events[instanceID].MetricSetFields.Put("instance.private.ip", *privateIP) - } + publicIP := instanceOutput[instanceID].PublicIpAddress + if publicIP != nil { + events[instanceID].MetricSetFields.Put("instance.public.ip", *publicIP) + } - events[instanceID].MetricSetFields.Put("instance.image.id", *instanceOutput[instanceID].ImageId) - events[instanceID].MetricSetFields.Put("instance.state.name", instanceStateName) - events[instanceID].MetricSetFields.Put("instance.state.code", *instanceOutput[instanceID].State.Code) - events[instanceID].MetricSetFields.Put("instance.monitoring.state", monitoringState) - events[instanceID].MetricSetFields.Put("instance.public.dns_name", *instanceOutput[instanceID].PublicDnsName) - events[instanceID].MetricSetFields.Put("instance.private.dns_name", *instanceOutput[instanceID].PrivateDnsName) + privateIP := instanceOutput[instanceID].PrivateIpAddress + if privateIP != nil { + events[instanceID].MetricSetFields.Put("instance.private.ip", *privateIP) } + + events[instanceID].MetricSetFields.Put("instance.image.id", *instanceOutput[instanceID].ImageId) + events[instanceID].MetricSetFields.Put("instance.state.name", instanceStateName) + events[instanceID].MetricSetFields.Put("instance.state.code", *instanceOutput[instanceID].State.Code) + events[instanceID].MetricSetFields.Put("instance.monitoring.state", monitoringState) + events[instanceID].MetricSetFields.Put("instance.public.dns_name", *instanceOutput[instanceID].PublicDnsName) + events[instanceID].MetricSetFields.Put("instance.private.dns_name", *instanceOutput[instanceID].PrivateDnsName) } } diff --git a/x-pack/metricbeat/module/aws/rds/rds.go b/x-pack/metricbeat/module/aws/rds/rds.go index f8bd907b3f6..3bb14f28de8 100644 --- a/x-pack/metricbeat/module/aws/rds/rds.go +++ b/x-pack/metricbeat/module/aws/rds/rds.go @@ -79,7 +79,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // of an error set the Error field of mb.Event or simply call report.Error(). func (m *MetricSet) Fetch(report mb.ReporterV2) error { // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() @@ -273,70 +274,72 @@ func (m *MetricSet) createCloudWatchEvents(getMetricDataResults []cloudwatch.Met // Find a timestamp for all metrics in output timestamp := aws.FindTimestamp(getMetricDataResults) - if !timestamp.IsZero() { - for _, output := range getMetricDataResults { - if len(output.Values) == 0 { - continue + if timestamp.IsZero() { + return nil, nil + } + + for _, output := range getMetricDataResults { + if len(output.Values) == 0 { + continue + } + exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) + if exists { + labels := strings.Split(*output.Label, " ") + // Collect dimension values from the labels and initialize events and metricSetFieldResults with dimValues + var dimValues string + for i := 1; i < len(labels); i += 2 { + dimValues = dimValues + labels[i+1] } - exists, timestampIdx := aws.CheckTimestampInArray(timestamp, output.Timestamps) - if exists { - labels := strings.Split(*output.Label, " ") - // Collect dimension values from the labels and initialize events and metricSetFieldResults with dimValues - var dimValues string - for i := 1; i < len(labels); i += 2 { - dimValues = dimValues + labels[i+1] - } - if _, ok := events[dimValues]; !ok { - events[dimValues] = aws.InitEvent(regionName, m.AccountName, m.AccountID) - } + if _, ok := events[dimValues]; !ok { + events[dimValues] = aws.InitEvent(regionName, m.AccountName, m.AccountID, timestamp) + } - if _, ok := metricSetFieldResults[dimValues]; !ok { - metricSetFieldResults[dimValues] = map[string]interface{}{} - } + if _, ok := metricSetFieldResults[dimValues]; !ok { + metricSetFieldResults[dimValues] = map[string]interface{}{} + } - if len(output.Values) > timestampIdx && len(labels) > 0 { - if labels[metricNameIdx] == "CPUUtilization" { - metricSetFieldResults[dimValues][labels[metricNameIdx]] = fmt.Sprint(output.Values[timestampIdx] / 100) - } else { - metricSetFieldResults[dimValues][labels[metricNameIdx]] = fmt.Sprint(output.Values[timestampIdx]) - } + if len(output.Values) > timestampIdx && len(labels) > 0 { + if labels[metricNameIdx] == "CPUUtilization" { + metricSetFieldResults[dimValues][labels[metricNameIdx]] = fmt.Sprint(output.Values[timestampIdx] / 100) + } else { + metricSetFieldResults[dimValues][labels[metricNameIdx]] = fmt.Sprint(output.Values[timestampIdx]) + } - for i := 1; i < len(labels); i += 2 { - if labels[i] == "DBInstanceIdentifier" { - dbIdentifier := labels[i+1] - if _, found := events[dbIdentifier]; found { - if _, found := dbInstanceMap[dbIdentifier]; !found { - delete(metricSetFieldResults, dimValues) - continue - } - events[dbIdentifier].RootFields.Put("cloud.availability_zone", dbInstanceMap[dbIdentifier].dbAvailabilityZone) - events[dbIdentifier].MetricSetFields.Put("db_instance.arn", dbInstanceMap[dbIdentifier].dbArn) - events[dbIdentifier].MetricSetFields.Put("db_instance.class", dbInstanceMap[dbIdentifier].dbClass) - events[dbIdentifier].MetricSetFields.Put("db_instance.identifier", dbInstanceMap[dbIdentifier].dbIdentifier) - events[dbIdentifier].MetricSetFields.Put("db_instance.status", dbInstanceMap[dbIdentifier].dbStatus) - - for _, tag := range dbInstanceMap[dbIdentifier].tags { - events[dbIdentifier].ModuleFields.Put("tags."+tag.Key, tag.Value) - } + for i := 1; i < len(labels); i += 2 { + if labels[i] == "DBInstanceIdentifier" { + dbIdentifier := labels[i+1] + if _, found := events[dbIdentifier]; found { + if _, found := dbInstanceMap[dbIdentifier]; !found { + delete(metricSetFieldResults, dimValues) + continue + } + events[dbIdentifier].RootFields.Put("cloud.availability_zone", dbInstanceMap[dbIdentifier].dbAvailabilityZone) + events[dbIdentifier].MetricSetFields.Put("db_instance.arn", dbInstanceMap[dbIdentifier].dbArn) + events[dbIdentifier].MetricSetFields.Put("db_instance.class", dbInstanceMap[dbIdentifier].dbClass) + events[dbIdentifier].MetricSetFields.Put("db_instance.identifier", dbInstanceMap[dbIdentifier].dbIdentifier) + events[dbIdentifier].MetricSetFields.Put("db_instance.status", dbInstanceMap[dbIdentifier].dbStatus) + + for _, tag := range dbInstanceMap[dbIdentifier].tags { + events[dbIdentifier].ModuleFields.Put("tags."+tag.Key, tag.Value) } } - metricSetFieldResults[dimValues][labels[i]] = fmt.Sprint(labels[(i + 1)]) + } + metricSetFieldResults[dimValues][labels[i]] = fmt.Sprint(labels[(i + 1)]) + } + + // if tags_filter is given, then only return metrics with DBInstanceIdentifier as dimension + if m.TagsFilter != nil { + if len(labels) == 1 { + delete(events, dimValues) + delete(metricSetFieldResults, dimValues) } - // if tags_filter is given, then only return metrics with DBInstanceIdentifier as dimension - if m.TagsFilter != nil { - if len(labels) == 1 { + for i := 1; i < len(labels); i += 2 { + if labels[i] != "DBInstanceIdentifier" && i == len(labels)-2 { delete(events, dimValues) delete(metricSetFieldResults, dimValues) } - - for i := 1; i < len(labels); i += 2 { - if labels[i] != "DBInstanceIdentifier" && i == len(labels)-2 { - delete(events, dimValues) - delete(metricSetFieldResults, dimValues) - } - } } } } diff --git a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go index 7c9d453baca..d5efa36fb03 100644 --- a/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go +++ b/x-pack/metricbeat/module/aws/s3_daily_storage/s3_daily_storage.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { @@ -185,8 +186,6 @@ func createMetricDataQuery(metric cloudwatch.Metric, period time.Duration, index } func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string, accountName string, accountID string) (event mb.Event, err error) { - event = aws.InitEvent(regionName, accountName, accountID) - // AWS s3_daily_storage metrics mapOfMetricSetFieldResults := make(map[string]interface{}) @@ -213,6 +212,7 @@ func createCloudWatchEvents(outputs []cloudwatch.MetricDataResult, regionName st return } + event = aws.InitEvent(regionName, accountName, accountID, timestamp) event.MetricSetFields = resultMetricSetFields event.RootFields.Put("aws.s3.bucket.name", bucketName) return diff --git a/x-pack/metricbeat/module/aws/s3_request/s3_request.go b/x-pack/metricbeat/module/aws/s3_request/s3_request.go index afe53ac49ba..00b82827bbf 100644 --- a/x-pack/metricbeat/module/aws/s3_request/s3_request.go +++ b/x-pack/metricbeat/module/aws/s3_request/s3_request.go @@ -69,7 +69,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/S3" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) // GetMetricData for AWS S3 from Cloudwatch for _, regionName := range m.MetricSet.RegionsList { @@ -187,8 +188,6 @@ func constructMetricQueries(listMetricsOutputs []cloudwatch.Metric, period time. // CreateS3Events creates s3_request and s3_daily_storage events from Cloudwatch metric data. func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName string, bucketName string, accountName string, accountID string) (event mb.Event, err error) { - event = aws.InitEvent(regionName, accountName, accountID) - // AWS s3_request metrics mapOfMetricSetFieldResults := make(map[string]interface{}) @@ -215,6 +214,7 @@ func createS3RequestEvents(outputs []cloudwatch.MetricDataResult, regionName str return } + event = aws.InitEvent(regionName, accountName, accountID, timestamp) event.MetricSetFields = resultMetricSetFields event.RootFields.Put("aws.s3.bucket.name", bucketName) return diff --git a/x-pack/metricbeat/module/aws/sqs/sqs.go b/x-pack/metricbeat/module/aws/sqs/sqs.go index 7bc6a8349d0..5f17eccb4b1 100644 --- a/x-pack/metricbeat/module/aws/sqs/sqs.go +++ b/x-pack/metricbeat/module/aws/sqs/sqs.go @@ -67,7 +67,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Fetch(report mb.ReporterV2) error { namespace := "AWS/SQS" // Get startTime and endTime - startTime, endTime := aws.GetStartTimeEndTime(m.Period) + startTime, endTime := aws.GetStartTimeEndTime(m.Period, m.Latency) + m.Logger().Debugf("startTime = %s, endTime = %s", startTime, endTime) for _, regionName := range m.MetricSet.RegionsList { awsConfig := m.MetricSet.AwsConfig.Copy() @@ -174,9 +175,7 @@ func createMetricDataQuery(metric cloudwatch.Metric, index int, period time.Dura return } -func createEventPerQueue(getMetricDataResults []cloudwatch.MetricDataResult, queueName string, metricsetName string, regionName string, schemaMetricFields s.Schema, accountName string, accountID string) (event mb.Event, err error) { - event = aws.InitEvent(regionName, accountName, accountID) - +func createEventPerQueue(getMetricDataResults []cloudwatch.MetricDataResult, queueName string, regionName string, schemaMetricFields s.Schema, accountName string, accountID string) (event mb.Event, err error) { // AWS sqs metrics mapOfMetricSetFieldResults := make(map[string]interface{}) @@ -203,6 +202,7 @@ func createEventPerQueue(getMetricDataResults []cloudwatch.MetricDataResult, que return } + event = aws.InitEvent(regionName, accountName, accountID, timestamp) event.MetricSetFields = resultMetricSetFields event.MetricSetFields.Put("queue.name", queueName) return @@ -212,7 +212,7 @@ func createSQSEvents(queueURLs []string, metricDataResults []cloudwatch.MetricDa for _, queueURL := range queueURLs { queueURLParsed := strings.Split(queueURL, "/") queueName := queueURLParsed[len(queueURLParsed)-1] - event, err := createEventPerQueue(metricDataResults, queueName, metricsetName, regionName, schemaRequestFields, accountName, accountID) + event, err := createEventPerQueue(metricDataResults, queueName, regionName, schemaRequestFields, accountName, accountID) if err != nil { event.Error = err report.Event(event) diff --git a/x-pack/metricbeat/module/aws/utils.go b/x-pack/metricbeat/module/aws/utils.go index 67e5809bc8e..d1846083854 100644 --- a/x-pack/metricbeat/module/aws/utils.go +++ b/x-pack/metricbeat/module/aws/utils.go @@ -22,8 +22,13 @@ import ( ) // GetStartTimeEndTime function uses durationString to create startTime and endTime for queries. -func GetStartTimeEndTime(period time.Duration) (time.Time, time.Time) { +func GetStartTimeEndTime(period time.Duration, latency time.Duration) (time.Time, time.Time) { endTime := time.Now() + if latency != 0 { + // add latency if config is not 0 + endTime = endTime.Add(latency * -1) + } + // Set startTime double the period earlier than the endtime in order to // make sure GetMetricDataRequest gets the latest data point for each metric. return endTime.Add(period * -2), endTime diff --git a/x-pack/metricbeat/module/aws/utils_test.go b/x-pack/metricbeat/module/aws/utils_test.go index aef35f57e61..3c480d347db 100644 --- a/x-pack/metricbeat/module/aws/utils_test.go +++ b/x-pack/metricbeat/module/aws/utils_test.go @@ -176,7 +176,7 @@ func TestGetListMetricsOutputWithWildcard(t *testing.T) { } func TestGetMetricDataPerRegion(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} var metricDataQueries []cloudwatch.MetricDataQuery @@ -205,7 +205,7 @@ func TestGetMetricDataPerRegion(t *testing.T) { } func TestGetMetricDataResults(t *testing.T) { - startTime, endTime := GetStartTimeEndTime(10 * time.Minute) + startTime, endTime := GetStartTimeEndTime(10*time.Minute, 0) mockSvc := &MockCloudWatchClient{} metricInfo := cloudwatch.Metric{ diff --git a/x-pack/metricbeat/module/azure/_meta/config.reference.yml b/x-pack/metricbeat/module/azure/_meta/config.reference.yml index 1f9ac04529e..b06e466a01f 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.reference.yml @@ -102,3 +102,11 @@ api_key: '' metrics: - id: ["requests/count", "requests/duration"] + +- module: azure + metricsets: + - app_state + enabled: true + period: 300s + application_id: '' + api_key: '' diff --git a/x-pack/metricbeat/module/azure/_meta/config.yml b/x-pack/metricbeat/module/azure/_meta/config.yml index 0f497af6fb4..f7215d4f991 100644 --- a/x-pack/metricbeat/module/azure/_meta/config.yml +++ b/x-pack/metricbeat/module/azure/_meta/config.yml @@ -111,3 +111,11 @@ # api_key: '' # metrics: # - id: ["requests/count", "requests/duration"] + +#- module: azure +# metricsets: +# - app_state +# enabled: true +# period: 300s +# application_id: '' +# api_key: '' diff --git a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc index b0f76ecb623..01f6389ab93 100644 --- a/x-pack/metricbeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/_meta/docs.asciidoc @@ -37,6 +37,10 @@ The Azure billing dashboards show relevant usage and forecast information: image::./images/metricbeat-azure-billing-overview.png[] +The Azure app_state dashboard shows relevant application insights information: + +image::./images/metricbeat-azure-app-state-overview.png[] + [float] === Module-specific configuration notes @@ -112,6 +116,10 @@ so the `period` for `billing` metricset should be `24h` or multiples of `24h`. === `app_insights` This metricset will collect application insights metrics, the `period` (interval) for the `app-insights` metricset is set by default at `300s`. +[float] +=== `app_state` +This metricset concentrate on the most relevant application insights metrics and provides a dashboard for visualization, the `period` (interval) for the `app_state` metricset is set by default at `300s`. + [float] [[azure-api-cost]] == Additional notes about metrics and costs diff --git a/x-pack/metricbeat/module/azure/_meta/fields.yml b/x-pack/metricbeat/module/azure/_meta/fields.yml index c6471dc108d..a6476fcc957 100644 --- a/x-pack/metricbeat/module/azure/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/_meta/fields.yml @@ -39,6 +39,10 @@ type: keyword description: > The subscription ID + - name: application_id + type: keyword + description: > + The application ID - name: dimensions.* type: object object_type: keyword diff --git a/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json new file mode 100644 index 00000000000..b447a6b276e --- /dev/null +++ b/x-pack/metricbeat/module/azure/_meta/kibana/7/dashboard/Metricbeat-azure-app-state-overview.json @@ -0,0 +1,1509 @@ +{ + "objects": [ + { + "attributes": { + "description": "Provides relevant app insights metrics for web applications", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "controlledBy": "1532342651170", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "azure.app_state.application_id", + "negate": false, + "params": { + "query": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "azure.app_state.application_id": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "307a1ecd-284c-4f35-9a3c-d5b77c9a9c82", + "w": 7, + "x": 0, + "y": 0 + }, + "panelIndex": "307a1ecd-284c-4f35-9a3c-d5b77c9a9c82", + "panelRefName": "panel_0", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Exceptions" + }, + "gridData": { + "h": 15, + "i": "654e745f-360d-4898-89b6-57f788c5f540", + "w": 20, + "x": 7, + "y": 0 + }, + "panelIndex": "654e745f-360d-4898-89b6-57f788c5f540", + "panelRefName": "panel_1", + "title": "Exceptions", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Available Memory" + }, + "gridData": { + "h": 15, + "i": "5adca737-559d-4b4f-9fa7-58841daa99c5", + "w": 21, + "x": 27, + "y": 0 + }, + "panelIndex": "5adca737-559d-4b4f-9fa7-58841daa99c5", + "panelRefName": "panel_2", + "title": "Available Memory", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "531cf244-45e0-43c3-9920-8f32397bd973", + "w": 8, + "x": 0, + "y": 15 + }, + "panelIndex": "531cf244-45e0-43c3-9920-8f32397bd973", + "panelRefName": "panel_3", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "b9242495-babc-48a7-9ad7-56c62b1dc117", + "w": 8, + "x": 8, + "y": 15 + }, + "panelIndex": "b9242495-babc-48a7-9ad7-56c62b1dc117", + "panelRefName": "panel_4", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "" + }, + "gridData": { + "h": 15, + "i": "d311025a-f5c5-4e48-9f1c-710f59264c43", + "w": 8, + "x": 16, + "y": 15 + }, + "panelIndex": "d311025a-f5c5-4e48-9f1c-710f59264c43", + "panelRefName": "panel_5", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Requests" + }, + "gridData": { + "h": 15, + "i": "48974418-b1f7-4050-921e-a83771e125ae", + "w": 24, + "x": 24, + "y": 15 + }, + "panelIndex": "48974418-b1f7-4050-921e-a83771e125ae", + "panelRefName": "panel_6", + "title": "Requests", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Browser Send/Receive Duration" + }, + "gridData": { + "h": 15, + "i": "39d20db1-316a-4ff3-811a-5571cb4497c3", + "w": 24, + "x": 0, + "y": 30 + }, + "panelIndex": "39d20db1-316a-4ff3-811a-5571cb4497c3", + "panelRefName": "panel_7", + "title": "Browser Send/Receive Duration", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Browser Networking/Processing Duration" + }, + "gridData": { + "h": 15, + "i": "bc810208-0395-4c70-9057-d7307e064e43", + "w": 24, + "x": 24, + "y": 30 + }, + "panelIndex": "bc810208-0395-4c70-9057-d7307e064e43", + "panelRefName": "panel_8", + "title": "Browser Networking/Processing Duration", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Process CPU Usage" + }, + "gridData": { + "h": 15, + "i": "ecf6fbfa-ba65-481e-af85-07fd9d5feb5f", + "w": 24, + "x": 0, + "y": 45 + }, + "panelIndex": "ecf6fbfa-ba65-481e-af85-07fd9d5feb5f", + "panelRefName": "panel_9", + "title": "Process CPU Usage", + "version": "7.9.2" + }, + { + "embeddableConfig": { + "title": "Process Private Bytes" + }, + "gridData": { + "h": 15, + "i": "40a1b80b-cd62-446d-91aa-a971bb3769e7", + "w": 24, + "x": 24, + "y": 45 + }, + "panelIndex": "40a1b80b-cd62-446d-91aa-a971bb3769e7", + "panelRefName": "panel_10", + "title": "Process Private Bytes", + "version": "7.9.2" + } + ], + "timeRestore": false, + "title": "[Metricbeat Azure] App State Overview", + "version": 1 + }, + "id": "d5fbd610-03d9-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "2e5183a0-03da-11eb-8034-63f2039e9d3f", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "1064f9a0-04a5-11eb-8034-63f2039e9d3f", + "name": "panel_1", + "type": "lens" + }, + { + "id": "76cc1d70-04a7-11eb-8034-63f2039e9d3f", + "name": "panel_2", + "type": "lens" + }, + { + "id": "a89c8fd0-03ec-11eb-8034-63f2039e9d3f", + "name": "panel_3", + "type": "lens" + }, + { + "id": "cb5ec410-03ed-11eb-8034-63f2039e9d3f", + "name": "panel_4", + "type": "lens" + }, + { + "id": "0df175c0-03ee-11eb-8034-63f2039e9d3f", + "name": "panel_5", + "type": "lens" + }, + { + "id": "f0678020-04a2-11eb-8034-63f2039e9d3f", + "name": "panel_6", + "type": "lens" + }, + { + "id": "e2704140-04a3-11eb-8034-63f2039e9d3f", + "name": "panel_7", + "type": "lens" + }, + { + "id": "0e74dee0-04a4-11eb-8034-63f2039e9d3f", + "name": "panel_8", + "type": "lens" + }, + { + "id": "cfa361a0-04a8-11eb-8034-63f2039e9d3f", + "name": "panel_9", + "type": "lens" + }, + { + "id": "2b54b2c0-04a8-11eb-8034-63f2039e9d3f", + "name": "panel_10", + "type": "lens" + } + ], + "type": "dashboard", + "updated_at": "2020-10-02T12:22:06.090Z", + "version": "WzMzMDEsMl0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [ + { + "$state": { + "store": "appState" + }, + "meta": { + "alias": null, + "controlledBy": "1532342651170", + "disabled": false, + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "key": "azure.application_id", + "negate": false, + "params": { + "query": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + }, + "type": "phrase" + }, + "query": { + "match_phrase": { + "azure.application_id": "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + } + } + } + ], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "App State Filters [Metricbeat Azure]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "controls": [ + { + "fieldName": "azure.application_id", + "id": "1532342651170", + "indexPatternRefName": "control_0_index_pattern", + "label": "Application ID", + "options": { + "multiselect": true, + "order": "desc", + "size": 10, + "type": "terms" + }, + "parent": "", + "type": "list" + }, + { + "fieldName": "azure.dimensions.request_url_host", + "id": "1601559750853", + "indexPatternRefName": "control_1_index_pattern", + "label": "Host URL", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + }, + { + "fieldName": "azure.dimensions.cloud_role_name", + "id": "1601640368472", + "indexPatternRefName": "control_2_index_pattern", + "label": "Name", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + }, + { + "fieldName": "azure.dimensions.browser_timing_url_host", + "id": "1601640439434", + "indexPatternRefName": "control_3_index_pattern", + "label": "Browser URL Host", + "options": { + "dynamicOptions": true, + "multiselect": true, + "order": "desc", + "size": 5, + "type": "terms" + }, + "parent": "1532342651170", + "type": "list" + } + ], + "pinFilters": false, + "updateFiltersOnChange": true, + "useTimeFilter": false + }, + "title": "App State Filters [Metricbeat Azure]", + "type": "input_control_vis" + } + }, + "id": "2e5183a0-03da-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "visualization": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_0_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_1_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_2_index_pattern", + "type": "index-pattern" + }, + { + "id": "metricbeat-*", + "name": "control_3_index_pattern", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-02T12:22:51.232Z", + "version": "WzMzMTIsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"5788331a-267d-426a-a68e-94a5310af644\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.exception_type\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_count.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_server.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.exceptions_browser.sum\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-4-5788331a-267d-426a-a68e-94a5310af644\\\":{\\\"label\\\":\\\"Type\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.exception_type\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"5788331a-267d-426a-a68e-94a5310af644\\\"},\\\"col-5-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Total\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_count.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"col-6-e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\":{\\\"label\\\":\\\"Server \\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_server.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\"},\\\"col-7-9e183a5e-3dba-4929-b07e-2a3321f7926b\\\":{\\\"label\\\":\\\"Browser\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.exceptions_browser.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"5788331a-267d-426a-a68e-94a5310af644\" seriesType=\"area_stacked\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" accessors=\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\" accessors=\"9e183a5e-3dba-4929-b07e-2a3321f7926b\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Total\\\",\\\"e5c93c50-bb0a-4609-a7ce-7003f2f9a20e\\\":\\\"Server \\\",\\\"9e183a5e-3dba-4929-b07e-2a3321f7926b\\\":\\\"Browser\\\",\\\"5788331a-267d-426a-a68e-94a5310af644\\\":\\\"Type\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "5788331a-267d-426a-a68e-94a5310af644", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e", + "9e183a5e-3dba-4929-b07e-2a3321f7926b" + ], + "columns": { + "5788331a-267d-426a-a68e-94a5310af644": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Type", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.exception_type" + }, + "9e183a5e-3dba-4929-b07e-2a3321f7926b": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Browser", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_browser.sum" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_count.sum" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + }, + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Server ", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.exceptions_server.sum" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "e5c93c50-bb0a-4609-a7ce-7003f2f9a20e", + "9e183a5e-3dba-4929-b07e-2a3321f7926b" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area_stacked", + "showGridlines": false, + "splitAccessor": "5788331a-267d-426a-a68e-94a5310af644", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area_stacked" + } + }, + "title": "App state Exceptions [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "1064f9a0-04a5-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:53:16.483Z", + "version": "WzI5NjMsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_memory_available_bytes.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-2-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-3-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Available memory\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_memory_available_bytes.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"bytes\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"}}\" | lens_format_column format=\"bytes\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Available memory\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Available memory\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "columns": { + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Available memory", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_memory_available_bytes.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Memory [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "76cc1d70-04a7-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:04:46.406Z", + "version": "WzMwNTcsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.users_count.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique users\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.users_count.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique users\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique users", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.users_count.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique users [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "a89c8fd0-03ec-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:47:34.093Z", + "version": "WzEzMzAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.users_authenticated.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique authenticated users\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.users_authenticated.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique authenticated users\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique authenticated users", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.users_authenticated.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique authenticated users [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "cb5ec410-03ed-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:55:41.904Z", + "version": "WzE0MjAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"82e648a8-6d9a-4ae0-9449-b802ce1ac723\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true aggConfigs=\"[{\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.sessions_count.unique\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\":{\\\"label\\\":\\\"Unique sessions\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.sessions_count.unique\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\\\"}}\"}\n| lens_metric_chart title=\"Unique sessions\" accessor=\"d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1\" mode=\"full\"", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "82e648a8-6d9a-4ae0-9449-b802ce1ac723": { + "columnOrder": [ + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1" + ], + "columns": { + "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Unique sessions", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.sessions_count.unique" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "accessor": "d62f1bf0-71b4-41ba-9d9c-dc9f4e478ac1", + "layerId": "82e648a8-6d9a-4ae0-9449-b802ce1ac723" + } + }, + "title": "App state Unique sessions [Metricbeat Azure]", + "visualizationType": "lnsMetric" + }, + "id": "0df175c0-03ee-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-01T13:57:33.596Z", + "version": "WzE0NjAsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.request_url_host\\\",\\\"orderBy\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.requests_count.sum\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.requests_failed.sum\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-8864c98b-413a-484f-a61d-336a63ef3f13\\\":{\\\"label\\\":\\\"Host URL\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.request_url_host\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\"},\\\"col-4-9ec4d260-e302-46c4-ac09-50ef54627894\\\":{\\\"label\\\":\\\"Total requests\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.requests_count.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\"},\\\"col-5-a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\":{\\\"label\\\":\\\"Failed requests\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.requests_failed.sum\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total requests\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"8864c98b-413a-484f-a61d-336a63ef3f13\" seriesType=\"area\" accessors=\"9ec4d260-e302-46c4-ac09-50ef54627894\" accessors=\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\" columnToLabel=\"{\\\"9ec4d260-e302-46c4-ac09-50ef54627894\\\":\\\"Total requests\\\",\\\"a47e59dc-fb62-42f8-90e1-236c7c5a073d\\\":\\\"Failed requests\\\",\\\"8864c98b-413a-484f-a61d-336a63ef3f13\\\":\\\"Host URL\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "8864c98b-413a-484f-a61d-336a63ef3f13", + "9ec4d260-e302-46c4-ac09-50ef54627894", + "a47e59dc-fb62-42f8-90e1-236c7c5a073d" + ], + "columns": { + "8864c98b-413a-484f-a61d-336a63ef3f13": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Host URL", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "9ec4d260-e302-46c4-ac09-50ef54627894", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.request_url_host" + }, + "9ec4d260-e302-46c4-ac09-50ef54627894": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total requests", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.requests_count.sum" + }, + "a47e59dc-fb62-42f8-90e1-236c7c5a073d": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Failed requests", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.requests_failed.sum" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "9ec4d260-e302-46c4-ac09-50ef54627894", + "a47e59dc-fb62-42f8-90e1-236c7c5a073d" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "8864c98b-413a-484f-a61d-336a63ef3f13", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Requests [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "f0678020-04a2-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:32:22.946Z", + "version": "WzI0MzUsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"orderBy\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_send_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_total_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_receive_duration.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-4-4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":{\\\"label\\\":\\\"Url Path\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\"},\\\"col-5-be6a3d8b-9428-480b-a7b3-071127726093\\\":{\\\"label\\\":\\\"Send duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_send_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"params\\\":{},\\\"customLabel\\\":true,\\\"id\\\":\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\"},\\\"col-6-6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\":{\\\"label\\\":\\\"Total duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_total_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\"},\\\"col-7-988e9976-3471-478c-89f6-11fd46458d7f\\\":{\\\"label\\\":\\\"Receive duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_receive_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Send duration\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"4d4c068a-0194-4d54-a1fa-3863c3df9331\" seriesType=\"bar\" accessors=\"be6a3d8b-9428-480b-a7b3-071127726093\" accessors=\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\" accessors=\"988e9976-3471-478c-89f6-11fd46458d7f\" columnToLabel=\"{\\\"be6a3d8b-9428-480b-a7b3-071127726093\\\":\\\"Send duration\\\",\\\"6bc1fd35-168d-42d5-b9c8-7078896d8ce4\\\":\\\"Total duration\\\",\\\"988e9976-3471-478c-89f6-11fd46458d7f\\\":\\\"Receive duration\\\",\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":\\\"Url Path\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "be6a3d8b-9428-480b-a7b3-071127726093", + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4", + "988e9976-3471-478c-89f6-11fd46458d7f" + ], + "columns": { + "4d4c068a-0194-4d54-a1fa-3863c3df9331": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Url Path", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "be6a3d8b-9428-480b-a7b3-071127726093", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.browser_timing_url_path" + }, + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_total_duration.avg" + }, + "988e9976-3471-478c-89f6-11fd46458d7f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Receive duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_receive_duration.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + }, + "be6a3d8b-9428-480b-a7b3-071127726093": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Send duration", + "operationType": "avg", + "params": {}, + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_send_duration.avg" + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "be6a3d8b-9428-480b-a7b3-071127726093", + "6bc1fd35-168d-42d5-b9c8-7078896d8ce4", + "988e9976-3471-478c-89f6-11fd46458d7f" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "bar", + "showGridlines": false, + "splitAccessor": "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar" + } + }, + "title": "App state Browser Send/Receive Duration [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "e2704140-04a3-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:39:09.012Z", + "version": "WzI2MzQsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"orderBy\\\":\\\"_key\\\",\\\"order\\\":\\\"asc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_network_duration.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.browser_timings_processing_duration.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":{\\\"label\\\":\\\"Url Path\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.browser_timing_url_path\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"alphabetical\\\"},\\\"orderDirection\\\":\\\"asc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\"},\\\"col-4-b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\":{\\\"label\\\":\\\"Networking duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_network_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\"},\\\"col-5-ab158cba-532f-47f8-8450-db883504dc0f\\\":{\\\"label\\\":\\\"Processing duration\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.browser_timings_processing_duration.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"id\\\":\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\"}}\"}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Networking duration\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"4d4c068a-0194-4d54-a1fa-3863c3df9331\" seriesType=\"bar\" accessors=\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\" accessors=\"ab158cba-532f-47f8-8450-db883504dc0f\" columnToLabel=\"{\\\"b5a75764-e98b-434b-a0f0-5658a4aa1cf6\\\":\\\"Networking duration\\\",\\\"ab158cba-532f-47f8-8450-db883504dc0f\\\":\\\"Processing duration\\\",\\\"4d4c068a-0194-4d54-a1fa-3863c3df9331\\\":\\\"Url Path\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6", + "ab158cba-532f-47f8-8450-db883504dc0f" + ], + "columns": { + "4d4c068a-0194-4d54-a1fa-3863c3df9331": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Url Path", + "operationType": "terms", + "params": { + "orderBy": { + "type": "alphabetical" + }, + "orderDirection": "asc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.browser_timing_url_path" + }, + "ab158cba-532f-47f8-8450-db883504dc0f": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Processing duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_processing_duration.avg" + }, + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Networking duration", + "operationType": "avg", + "scale": "ratio", + "sourceField": "azure.app_state.browser_timings_network_duration.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b5a75764-e98b-434b-a0f0-5658a4aa1cf6", + "ab158cba-532f-47f8-8450-db883504dc0f" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "bar", + "showGridlines": false, + "splitAccessor": "4d4c068a-0194-4d54-a1fa-3863c3df9331", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar" + } + }, + "title": "App state Browser Networking/Processing Duration [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "0e74dee0-04a4-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T11:40:22.862Z", + "version": "WzI2ODQsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage_total.avg\\\",\\\"missing\\\":0}},{\\\"id\\\":\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-3-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-4-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Total CPU percentage \\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage_total.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"percent\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"col-5-252dfd5f-26bd-4861-bb01-4b3530cadd95\\\":{\\\"label\\\":\\\"CPU percentage\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_cpu_percentage.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"percent\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"customLabel\\\":true,\\\"id\\\":\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\"}}\" | lens_format_column format=\"percent\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2 | lens_format_column format=\"percent\" columnId=\"252dfd5f-26bd-4861-bb01-4b3530cadd95\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Total CPU percentage \" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" accessors=\"252dfd5f-26bd-4861-bb01-4b3530cadd95\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Total CPU percentage \\\",\\\"252dfd5f-26bd-4861-bb01-4b3530cadd95\\\":\\\"CPU percentage\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "252dfd5f-26bd-4861-bb01-4b3530cadd95" + ], + "columns": { + "252dfd5f-26bd-4861-bb01-4b3530cadd95": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "CPU percentage", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_cpu_percentage.avg" + }, + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Total CPU percentage ", + "operationType": "avg", + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_cpu_percentage_total.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "252dfd5f-26bd-4861-bb01-4b3530cadd95" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Process CPU usage [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "cfa361a0-04a8-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:14:24.954Z", + "version": "WzMyNTMsMl0=" + }, + { + "attributes": { + "description": "", + "expression": "kibana\n| kibana_context query=\"{\\\"query\\\":\\\"\\\",\\\"language\\\":\\\"kuery\\\"}\" filters=\"[]\"\n| lens_merge_tables layerIds=\"85644d0a-8011-45af-a751-7961b8bdd071\" \n tables={esaggs index=\"metricbeat-*\" metricsAtAllLevels=true partialRows=true includeFormatHints=true timeFields=\"@timestamp\" aggConfigs=\"[{\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"date_histogram\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"@timestamp\\\",\\\"useNormalizedEsInterval\\\":true,\\\"interval\\\":\\\"auto\\\",\\\"drop_partials\\\":false,\\\"min_doc_count\\\":0,\\\"extended_bounds\\\":{}}},{\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"terms\\\",\\\"schema\\\":\\\"segment\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"orderBy\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"order\\\":\\\"desc\\\",\\\"size\\\":3,\\\"otherBucket\\\":false,\\\"otherBucketLabel\\\":\\\"Other\\\",\\\"missingBucket\\\":false,\\\"missingBucketLabel\\\":\\\"Missing\\\"}},{\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\",\\\"enabled\\\":true,\\\"type\\\":\\\"avg\\\",\\\"schema\\\":\\\"metric\\\",\\\"params\\\":{\\\"field\\\":\\\"azure.app_state.performance_counters_process_private_bytes.avg\\\",\\\"missing\\\":0}}]\" | lens_rename_columns idMap=\"{\\\"col-0-bcbccc16-d042-40fa-a9b2-0f09268281ff\\\":{\\\"label\\\":\\\"@timestamp\\\",\\\"dataType\\\":\\\"date\\\",\\\"operationType\\\":\\\"date_histogram\\\",\\\"suggestedPriority\\\":1,\\\"sourceField\\\":\\\"@timestamp\\\",\\\"isBucketed\\\":true,\\\"scale\\\":\\\"interval\\\",\\\"params\\\":{\\\"interval\\\":\\\"auto\\\"},\\\"id\\\":\\\"bcbccc16-d042-40fa-a9b2-0f09268281ff\\\"},\\\"col-2-a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":{\\\"label\\\":\\\"Instance\\\",\\\"dataType\\\":\\\"string\\\",\\\"operationType\\\":\\\"terms\\\",\\\"scale\\\":\\\"ordinal\\\",\\\"sourceField\\\":\\\"azure.dimensions.cloud_role_instance\\\",\\\"isBucketed\\\":true,\\\"params\\\":{\\\"size\\\":3,\\\"orderBy\\\":{\\\"type\\\":\\\"column\\\",\\\"columnId\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"},\\\"orderDirection\\\":\\\"desc\\\"},\\\"customLabel\\\":true,\\\"id\\\":\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\"},\\\"col-3-b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":{\\\"label\\\":\\\"Process private bytes\\\",\\\"dataType\\\":\\\"number\\\",\\\"operationType\\\":\\\"avg\\\",\\\"sourceField\\\":\\\"azure.app_state.performance_counters_process_private_bytes.avg\\\",\\\"isBucketed\\\":false,\\\"scale\\\":\\\"ratio\\\",\\\"customLabel\\\":true,\\\"params\\\":{\\\"format\\\":{\\\"id\\\":\\\"bytes\\\",\\\"params\\\":{\\\"decimals\\\":2}}},\\\"id\\\":\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\"}}\" | lens_format_column format=\"bytes\" columnId=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" decimals=2}\n| lens_xy_chart xTitle=\"@timestamp\" yTitle=\"Process private bytes\" legend={lens_xy_legendConfig isVisible=true position=\"right\"} fittingFunction=\"None\" \n layers={lens_xy_layer layerId=\"85644d0a-8011-45af-a751-7961b8bdd071\" hide=false xAccessor=\"bcbccc16-d042-40fa-a9b2-0f09268281ff\" yScaleType=\"linear\" xScaleType=\"time\" isHistogram=true splitAccessor=\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\" seriesType=\"area\" accessors=\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\" columnToLabel=\"{\\\"b0d8f2d4-91f3-469c-8bcb-962a9fb48fba\\\":\\\"Process private bytes\\\",\\\"a1f669d0-c9f2-4bc5-9bdd-e40badd261b9\\\":\\\"Instance\\\"}\"}", + "state": { + "datasourceMetaData": { + "filterableIndexPatterns": [ + { + "id": "metricbeat-*", + "title": "metricbeat-*" + } + ] + }, + "datasourceStates": { + "indexpattern": { + "currentIndexPatternId": "metricbeat-*", + "layers": { + "85644d0a-8011-45af-a751-7961b8bdd071": { + "columnOrder": [ + "bcbccc16-d042-40fa-a9b2-0f09268281ff", + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "columns": { + "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9": { + "customLabel": true, + "dataType": "string", + "isBucketed": true, + "label": "Instance", + "operationType": "terms", + "params": { + "orderBy": { + "columnId": "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba", + "type": "column" + }, + "orderDirection": "desc", + "size": 3 + }, + "scale": "ordinal", + "sourceField": "azure.dimensions.cloud_role_instance" + }, + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba": { + "customLabel": true, + "dataType": "number", + "isBucketed": false, + "label": "Process private bytes", + "operationType": "avg", + "params": { + "format": { + "id": "bytes", + "params": { + "decimals": 2 + } + } + }, + "scale": "ratio", + "sourceField": "azure.app_state.performance_counters_process_private_bytes.avg" + }, + "bcbccc16-d042-40fa-a9b2-0f09268281ff": { + "dataType": "date", + "isBucketed": true, + "label": "@timestamp", + "operationType": "date_histogram", + "params": { + "interval": "auto" + }, + "scale": "interval", + "sourceField": "@timestamp", + "suggestedPriority": 1 + } + }, + "indexPatternId": "metricbeat-*" + } + } + } + }, + "filters": [], + "query": { + "language": "kuery", + "query": "" + }, + "visualization": { + "fittingFunction": "None", + "layers": [ + { + "accessors": [ + "b0d8f2d4-91f3-469c-8bcb-962a9fb48fba" + ], + "layerId": "85644d0a-8011-45af-a751-7961b8bdd071", + "position": "top", + "seriesType": "area", + "showGridlines": false, + "splitAccessor": "a1f669d0-c9f2-4bc5-9bdd-e40badd261b9", + "xAccessor": "bcbccc16-d042-40fa-a9b2-0f09268281ff" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "area" + } + }, + "title": "App state Process Private Bytes [Metricbeat Azure]", + "visualizationType": "lnsXY" + }, + "id": "2b54b2c0-04a8-11eb-8034-63f2039e9d3f", + "migrationVersion": { + "lens": "7.8.0" + }, + "namespaces": [ + "default" + ], + "references": [], + "type": "lens", + "updated_at": "2020-10-02T12:09:49.291Z", + "version": "WzMxMjMsMl0=" + } + ], + "version": "7.9.2" +} diff --git a/x-pack/metricbeat/module/azure/add_metadata.go b/x-pack/metricbeat/module/azure/add_metadata.go index ba8f35c7db6..1342170aec0 100644 --- a/x-pack/metricbeat/module/azure/add_metadata.go +++ b/x-pack/metricbeat/module/azure/add_metadata.go @@ -35,7 +35,7 @@ func addHostMetadata(event *mb.Event, metricList common.MapStr) { } } -func addCloudVMMetadata(event *mb.Event, vm VmResource) { +func addCloudVMMetadata(event *mb.Event, vm VmResource, subscriptionId string) { if vm.Name != "" { event.RootFields.Put("cloud.instance.name", vm.Name) event.RootFields.Put("host.name", vm.Name) @@ -47,4 +47,7 @@ func addCloudVMMetadata(event *mb.Event, vm VmResource) { if vm.Size != "" { event.RootFields.Put("cloud.machine.type", vm.Size) } + if subscriptionId != "" { + event.RootFields.Put("cloud.account.id", subscriptionId) + } } diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/data.json b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json index 3d1f07c1ac1..c9b35f39923 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/data.json +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/data.json @@ -1,16 +1,15 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "azure" : { - "app_insights" : { - "metrics" : { - "requests_failed" : { - "sum" : 182 - }, - "request_name" : "GET /favicon.ico" - }, - "start_date" : "2020-07-12T10:52:11.831Z", - "end_date" : "2020-07-12T12:52:11.831Z", - "application_id" : "42cb59a9-d5be-400b-a5c4-69b0a0026ac6" + "azure": { + "app_insights": { + "end_date": "2020-10-02T13:17:45.691Z", + "start_date": "2020-10-02T13:12:45.691Z" + }, + "application_id": "42cb59a9-d5be-400b-a5c4-69b0a00434fdf4", + "metrics": { + "requests_count": { + "sum": 0 + } } }, "cloud": { diff --git a/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml index 40ab8560827..4e8fa91edbd 100644 --- a/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml +++ b/x-pack/metricbeat/module/azure/app_insights/_meta/fields.yml @@ -4,10 +4,6 @@ description: > application insights fields: - - name: application_id - type: keyword - description: > - The application ID - name: start_date type: date description: > diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights.go b/x-pack/metricbeat/module/azure/app_insights/app_insights.go index 8ffe02eb860..091a72f465e 100644 --- a/x-pack/metricbeat/module/azure/app_insights/app_insights.go +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights.go @@ -23,6 +23,7 @@ type Config struct { ApiKey string `config:"api_key" validate:"required"` Period time.Duration `config:"period" validate:"nonzero,required"` Metrics []Metric `config:"metrics" validate:"required"` + Namespace string `config:"namespace"` } // Metric struct used for configuration options @@ -70,7 +71,7 @@ func (m *MetricSet) Fetch(report mb.ReporterV2) error { if err != nil { return errors.Wrap(err, "error retrieving metric values") } - events := EventsMapping(results, m.client.Config.ApplicationId) + events := EventsMapping(results, m.client.Config.ApplicationId, m.client.Config.Namespace) for _, event := range events { isOpen := report.Event(event) if !isOpen { diff --git a/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go index 3cb93663007..c33f05c2de6 100644 --- a/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go +++ b/x-pack/metricbeat/module/azure/app_insights/app_insights_integration_test.go @@ -10,12 +10,20 @@ package app_insights import ( "testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" ) +var metrics = []map[string]interface{}{{ + "id": "requests/count", +}} + func TestFetchMetricset(t *testing.T) { + config := test.GetConfigForInsights(t, "app_insights") + config["metrics"] = metrics metricSet := mbtest.NewReportingMetricSetV2Error(t, config) events, errs := mbtest.ReportingFetchV2Error(metricSet) if len(errs) > 0 { @@ -26,6 +34,8 @@ func TestFetchMetricset(t *testing.T) { } func TestData(t *testing.T) { + config := test.GetConfigForInsights(t, "app_insights") + config["metrics"] = metrics metricSet := mbtest.NewFetcher(t, config) metricSet.WriteEvents(t, "/") } diff --git a/x-pack/metricbeat/module/azure/app_insights/client.go b/x-pack/metricbeat/module/azure/app_insights/client.go index b78f2257d3f..49794da4d3f 100644 --- a/x-pack/metricbeat/module/azure/app_insights/client.go +++ b/x-pack/metricbeat/module/azure/app_insights/client.go @@ -23,9 +23,6 @@ type Client struct { Log *logp.Logger } -type MetricValue struct { -} - // NewClient instantiates the an Azure monitoring client func NewClient(config Config) (*Client, error) { service, err := NewService(config) diff --git a/x-pack/metricbeat/module/azure/app_insights/data.go b/x-pack/metricbeat/module/azure/app_insights/data.go index df7efdbeaba..1cd661cc3a0 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data.go +++ b/x-pack/metricbeat/module/azure/app_insights/data.go @@ -6,98 +6,217 @@ package app_insights import ( "fmt" + "regexp" "strings" + "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" "github.com/Azure/go-autorest/autorest/date" - "github.com/Azure/azure-sdk-for-go/services/preview/appinsights/v1/insights" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) -func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId string) []mb.Event { - var events []mb.Event - if metricValues.Value == nil { - return events - } - groupedAddProp := make(map[string][]insights.MetricsResultInfo) +const aggsRegex = "_(?:sum|count|unique|avg|min|max)$" + +// segmentNames list is used to filter out the dimension from the api response. Based on the body format it is not possible to detect what was the segment selected +var segmentNames = []string{ + "request_source", "request_name", "request_url_host", "request_url_path", "request_success", "request_result_code", "request_performance_bucket", "operation_name", "operation_synthetic", "operation_synthetic_source", "user_authenticated", "application_version", "client_type", "client_model", + "client_os", "client_city", "client_state_or_province", "client_country_or_region", "client_browser", "cloud_role_name", "cloud_role_instance", "custom_dimensions__ms_processedb_by_metric_extractors", "custom_dimensions_developer_mode", + "page_view_name", "page_view_url_path", "page_view_url_host", "page_view_performance_bucket", "custom_dimensions_ibiza_session_id", "custom_dimensions_part_instance", "browser_timing_name", "browser_timing_url_host", "browser_timing_url_path", "browser_timing_performance_bucket", + "trace_severity_level", "type", "custom_dimensions_agent_session", "custom_dimensions_agent_version", "custom_dimensions_machine_name", "custom_dimensions_running_mode", "custom_dimensions_source", "custom_dimensions_agent_assembly_version", "custom_dimensions_agent_process_session", + "custom_dimensions_hashed_machine_name", + "custom_dimensions_data_cube", "dependency_target", "dependency_type", "dependency_name", "dependency_success", "dependency_result_code", "dependency_performance_bucket", "custom_dimensions_container", "custom_dimensions_blob", "custom_dimensions_error_message", + "custom_event_name", "custom_dimensions_event_name", "custom_dimensions_page_title", "custom_dimensions_service_profiler_content", "custom_dimensions_executing_assembly_file_version", "custom_dimensions_service_profiler_version", "custom_dimensions_process_id", "custom_dimensions_request_id", + "custom_dimensions_running_session", "custom_dimensions_problem_id", "custom_dimensions_snapshot_context", "custom_dimensions_snapshot_version", "custom_dimensions_duration", "custom_dimensions_snapshot_id", "custom_dimensions_stamp_id", "custom_dimensions_de_optimization_id", + "custom_dimensions_method", "custom_dimensions_parent_process_id", "custom_dimensions_section", "custom_dimensions_configuration", "custom_dimensions_dump_folder", "custom_dimensions_reason", "custom_dimensions_extension_version", "custom_dimensions_site_name", + "availability_result_name", "availability_result_location", "availability_result_success", "custom_dimensions_full_test_result_available", "exception_problem_id", "exception_handled_at", "exception_type", "exception_assembly", "exception_method", "custom_dimensions_custom_perf_counter", + "exception_severity_level", "custom_dimensions_url", "custom_dimensions_ai.snapshot_stampid", "custom_dimensions_ai.snapshot_id", "custom_dimensions_ai.snapshot_version", "custom_dimensions_ai.snapshot_planid", "custom_dimensions__ms_example", "custom_dimensions_sa_origin_app_id", + "custom_dimensions_base_sdk_target_framework", "custom_dimensions_runtime_framework", "custom_dimensions__ms_aggregation_interval_ms", "custom_dimensions_problem_id", "custom_dimensions_operation_name", "custom_dimensions_request_success", "custom_dimensions__ms_metric_id", + "custom_dimensions_dependency_success", "custom_dimensions__ms_is_autocollected", "custom_dimensions_dependency_type", "performance_counter_name", "performance_counter_category", "performance_counter_counter", "performance_counter_instance", "custom_dimensions_counter_instance_name", +} + +type MetricValue struct { + SegmentName map[string]string + Value map[string]interface{} + Segments []MetricValue + Interval string + Start *date.Time + End *date.Time +} + +func mapMetricValues(metricValues insights.ListMetricsResultsItem) []MetricValue { + var mapped []MetricValue for _, item := range *metricValues.Value { + metricValue := MetricValue{ + Start: item.Body.Value.Start, + End: item.Body.Value.End, + Value: map[string]interface{}{}, + SegmentName: map[string]string{}, + } + metricValue.Interval = fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End) if item.Body != nil && item.Body.Value != nil { if item.Body.Value.AdditionalProperties != nil { - groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)] = - append(groupedAddProp[fmt.Sprintf("%sTO%s", item.Body.Value.Start, item.Body.Value.End)], *item.Body.Value) - } else if item.Body.Value.Segments != nil { - for _, segment := range *item.Body.Value.Segments { - event, ok := createSegmentEvent(*item.Body.Value.Start, *item.Body.Value.End, segment, applicationId) - if ok { - events = append(events, event) + metrics := getAdditionalPropMetric(item.Body.Value.AdditionalProperties) + for key, metric := range metrics { + if isSegment(key) { + metricValue.SegmentName[key] = metric.(string) + } else { + metricValue.Value[key] = metric } } } + if item.Body.Value.Segments != nil { + for _, segment := range *item.Body.Value.Segments { + metVal := mapSegment(segment, metricValue.SegmentName) + metricValue.Segments = append(metricValue.Segments, metVal) + } + } + mapped = append(mapped, metricValue) } + } - if len(groupedAddProp) > 0 { - for _, val := range groupedAddProp { - event, ok := createEvent(val, applicationId) - if ok { - events = append(events, event) + return mapped +} + +func mapSegment(segment insights.MetricsSegmentInfo, parentSeg map[string]string) MetricValue { + metricValue := MetricValue{Value: map[string]interface{}{}, SegmentName: map[string]string{}} + if segment.AdditionalProperties != nil { + metrics := getAdditionalPropMetric(segment.AdditionalProperties) + for key, metric := range metrics { + if isSegment(key) { + metricValue.SegmentName[key] = metric.(string) + } else { + metricValue.Value[key] = metric + } + } + } + if len(parentSeg) > 0 { + for key, val := range parentSeg { + metricValue.SegmentName[key] = val + } + } + if segment.Segments != nil { + for _, segment := range *segment.Segments { + metVal := mapSegment(segment, metricValue.SegmentName) + metricValue.Segments = append(metricValue.Segments, metVal) + } + } + + return metricValue +} + +func isSegment(metric string) bool { + for _, seg := range segmentNames { + if metric == seg { + return true + } + } + return false +} + +func EventsMapping(metricValues insights.ListMetricsResultsItem, applicationId string, namespace string) []mb.Event { + var events []mb.Event + if metricValues.Value == nil { + return events + } + groupedAddProp := make(map[string][]MetricValue) + mValues := mapMetricValues(metricValues) + + var segValues []MetricValue + for _, mv := range mValues { + if len(mv.Segments) == 0 { + groupedAddProp[mv.Interval] = append(groupedAddProp[mv.Interval], mv) + } else { + segValues = append(segValues, mv) + } + } + + for _, val := range groupedAddProp { + event := createNoSegEvent(val, applicationId, namespace) + if len(event.MetricSetFields) > 0 { + events = append(events, event) + } + } + for _, val := range segValues { + for _, seg := range val.Segments { + lastSeg := getValue(seg) + for _, ls := range lastSeg { + events = append(events, createSegEvent(val, ls, applicationId, namespace)) } } } return events } -func createSegmentEvent(start date.Time, end date.Time, segment insights.MetricsSegmentInfo, applicationId string) (mb.Event, bool) { - metricList := common.MapStr{} - metrics := getMetric(segment.AdditionalProperties) - if len(metrics) == 0 { - return mb.Event{}, false +func getValue(metric MetricValue) []MetricValue { + var values []MetricValue + if metric.Segments == nil { + return []MetricValue{metric} + } + for _, met := range metric.Segments { + values = append(values, getValue(met)...) } - for key, metric := range metrics { + return values +} + +func createSegEvent(parentMetricValue MetricValue, metricValue MetricValue, applicationId string, namespace string) mb.Event { + metricList := common.MapStr{} + for key, metric := range metricValue.Value { metricList.Put(key, metric) } + if len(metricList) == 0 { + return mb.Event{} + } + event := createEvent(parentMetricValue.Start, parentMetricValue.End, applicationId, namespace, metricList) + if len(parentMetricValue.SegmentName) > 0 { + event.ModuleFields.Put("dimensions", parentMetricValue.SegmentName) + } + if len(metricValue.SegmentName) > 0 { + event.ModuleFields.Put("dimensions", metricValue.SegmentName) + } + return event +} + +func createEvent(start *date.Time, end *date.Time, applicationId string, namespace string, metricList common.MapStr) mb.Event { event := mb.Event{ - MetricSetFields: common.MapStr{ - "start_date": start, - "end_date": end, + ModuleFields: common.MapStr{ "application_id": applicationId, }, + MetricSetFields: common.MapStr{ + "start_date": start, + "end_date": end, + }, Timestamp: end.Time, } event.RootFields = common.MapStr{} event.RootFields.Put("cloud.provider", "azure") - event.MetricSetFields.Put("metrics", metricList) - return event, true + if namespace == "" { + event.ModuleFields.Put("metrics", metricList) + } else { + for key, metric := range metricList { + event.MetricSetFields.Put(key, metric) + } + } + return event } -func createEvent(values []insights.MetricsResultInfo, applicationId string) (mb.Event, bool) { +func createNoSegEvent(values []MetricValue, applicationId string, namespace string) mb.Event { metricList := common.MapStr{} for _, value := range values { - metrics := getMetric(value.AdditionalProperties) - for key, metric := range metrics { + for key, metric := range value.Value { metricList.Put(key, metric) } } if len(metricList) == 0 { - return mb.Event{}, false + return mb.Event{} } + return createEvent(values[0].Start, values[0].End, applicationId, namespace, metricList) - event := mb.Event{ - MetricSetFields: common.MapStr{ - "start_date": values[0].Start, - "end_date": values[0].End, - "application_id": applicationId, - }, - Timestamp: values[0].End.Time, - } - event.RootFields = common.MapStr{} - event.RootFields.Put("cloud.provider", "azure") - event.MetricSetFields.Put("metrics", metricList) - return event, true } -func getMetric(addProp map[string]interface{}) map[string]interface{} { +func getAdditionalPropMetric(addProp map[string]interface{}) map[string]interface{} { metricNames := make(map[string]interface{}) for key, val := range addProp { switch val.(type) { @@ -115,5 +234,19 @@ func getMetric(addProp map[string]interface{}) map[string]interface{} { } func cleanMetricNames(metric string) string { - return strings.Replace(metric, "/", "_", -1) + metric = strings.Replace(metric, "/", "_", -1) + metric = strings.Replace(metric, " ", "_", -1) + metric = azure.ReplaceUpperCase(metric) + obj := strings.Split(metric, ".") + for index := range obj { + // in some cases a trailing "_" is found + obj[index] = strings.TrimPrefix(obj[index], "_") + obj[index] = strings.TrimSuffix(obj[index], "_") + } + metric = strings.ToLower(strings.Join(obj, "_")) + aggsRegex := regexp.MustCompile(aggsRegex) + metric = aggsRegex.ReplaceAllStringFunc(metric, func(str string) string { + return strings.Replace(str, "_", ".", -1) + }) + return metric } diff --git a/x-pack/metricbeat/module/azure/app_insights/data_test.go b/x-pack/metricbeat/module/azure/app_insights/data_test.go index ebe4e7d98aa..f65f9f84b1d 100644 --- a/x-pack/metricbeat/module/azure/app_insights/data_test.go +++ b/x-pack/metricbeat/module/azure/app_insights/data_test.go @@ -39,20 +39,32 @@ func TestEventMapping(t *testing.T) { Value: &metrics, } applicationId := "abc" - events := EventsMapping(result, applicationId) + events := EventsMapping(result, applicationId, "") assert.Equal(t, len(events), 1) for _, event := range events { val1, _ := event.MetricSetFields.GetValue("start_date") assert.Equal(t, val1, &startDate) val2, _ := event.MetricSetFields.GetValue("end_date") assert.Equal(t, val2, &startDate) - val3, _ := event.MetricSetFields.GetValue("metrics.requests_count") + val3, _ := event.ModuleFields.GetValue("metrics.requests_count") assert.Equal(t, val3, common.MapStr{"sum": 12}) - val5, _ := event.MetricSetFields.GetValue("metrics.requests_failed") + val5, _ := event.ModuleFields.GetValue("metrics.requests_failed") assert.Equal(t, val5, common.MapStr{"sum": 10}) - val4, _ := event.MetricSetFields.GetValue("application_id") + val4, _ := event.ModuleFields.GetValue("application_id") assert.Equal(t, val4, applicationId) } } + +func TestCleanMetricNames(t *testing.T) { + ex := "customDimensions/ExecutingAssemblyFileVersion" + result := cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions_executing_assembly_file_version") + ex = "customDimensions/_MS.AggregationIntervalMs" + result = cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions__ms_aggregation_interval_ms") + ex = "customDimensions/_MS.IsAutocollected" + result = cleanMetricNames(ex) + assert.Equal(t, result, "custom_dimensions__ms_is_autocollected") +} diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/data.json b/x-pack/metricbeat/module/azure/app_state/_meta/data.json new file mode 100644 index 00000000000..c4744db7f60 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/data.json @@ -0,0 +1,32 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "azure": { + "app_state": { + "end_date": "2020-10-02T13:23:11.221Z", + "requests_count": { + "sum": 16 + }, + "start_date": "2020-10-01T13:23:11.221Z" + }, + "application_id": "42cb59a9-d5be-400b-a5c4-69b0a00434fdf4", + "dimensions": { + "request_name": "GET /auth", + "request_url_host": "demoapplogobs.azurewebsites.net" + } + }, + "cloud": { + "provider": "azure" + }, + "event": { + "dataset": "azure.app_state", + "duration": 115000, + "module": "azure" + }, + "metricset": { + "name": "app_state", + "period": 10000 + }, + "service": { + "type": "azure" + } +} diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc new file mode 100644 index 00000000000..4517ccfed0d --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/docs.asciidoc @@ -0,0 +1,12 @@ +This is the app_state metricset of the module azure. + +This metricset allows users to retrieve application insights metrics from specified applications. + +[float] +==== Config options to identify resources + +`application_id`:: (_[]string_) ID of the application. This is Application ID from the API Access settings blade in the Azure portal. + +`api_key`:: (_[]string_) The API key which will be generated, more on the steps here https://dev.applicationinsights.io/documentation/Authorization/API-key-and-App-ID. + + diff --git a/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml b/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml new file mode 100644 index 00000000000..fad928c090f --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/_meta/fields.yml @@ -0,0 +1,86 @@ +- name: app_state + type: group + release: beta + description: > + application state + fields: + - name: start_date + type: date + description: > + The start date + - name: end_date + type: date + description: > + The end date + - name: requests_count.sum + type: float + description: > + Request count + - name: requests_failed.sum + type: float + description: > + Request failed count + - name: users_count.unique + type: float + description: > + User count + - name: sessions_count.unique + type: float + description: > + Session count + - name: users_authenticated.unique + type: float + description: > + Authenticated users count + - name: browser_timings_network_duration.avg + type: float + description: > + Browser timings network duration + - name: browser_timings_send_duration.avg + type: float + description: > + Browser timings send duration + - name: browser_timings_receive_uration.avg + type: float + description: > + Browser timings receive duration + - name: browser_timings_processing_duration.avg + type: float + description: > + Browser timings processing duration + - name: browser_timings_total_duration.avg + type: float + description: > + Browser timings total duration + - name: exceptions_count.sum + type: float + description: > + Exception count + - name: exceptions_browser.sum + type: float + description: > + Exception count at browser level + - name: exceptions_server.sum + type: float + description: > + Exception count at server level + - name: performance_counters_memory_available_bytes.avg + type: float + description: > + Performance counters memory available bytes + - name: performance_counters_process_private_bytes.avg + type: float + description: > + Performance counters process private bytes + - name: performance_counters_process_cpu_percentage_total.avg + type: float + description: > + Performance counters process cpu percentage total + - name: performance_counters_process_cpu_percentage.avg + type: float + description: > + Performance counters process cpu percentage + - name: performance_counters_processiobytes_per_second.avg + type: float + description: > + Performance counters process IO bytes per second diff --git a/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go b/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go new file mode 100644 index 00000000000..8762cae440d --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/app_state_integration_test.go @@ -0,0 +1,38 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build integration +// +build azure + +package app_state + +import ( + "testing" + + "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/test" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/azure/app_insights" +) + +func TestFetchMetricset(t *testing.T) { + config := test.GetConfigForInsights(t, "app_state") + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + +func TestData(t *testing.T) { + config := test.GetConfigForInsights(t, "app_state") + metricSet := mbtest.NewFetcher(t, config) + metricSet.WriteEvents(t, "/") +} diff --git a/x-pack/metricbeat/module/azure/app_state/app_state_test.go b/x-pack/metricbeat/module/azure/app_state/app_state_test.go new file mode 100644 index 00000000000..a8dbab88081 --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/app_state_test.go @@ -0,0 +1,17 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package app_state + +import ( + "os" + + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} diff --git a/x-pack/metricbeat/module/azure/app_state/manifest.yml b/x-pack/metricbeat/module/azure/app_state/manifest.yml new file mode 100644 index 00000000000..5c1d08a2ceb --- /dev/null +++ b/x-pack/metricbeat/module/azure/app_state/manifest.yml @@ -0,0 +1,29 @@ +default: false +input: + module: azure + metricset: app_insights + defaults: + namespace: app_state + metrics: + - id: ["requests/count", "requests/failed"] + segment: ["request/urlHost", "request/name"] + aggregation: ["sum"] + interval: "P5M" + - id: ["users/count", "sessions/count", "users/authenticated"] + segment: ["request/urlHost"] + aggregation: ["unique"] + interval: "P5M" + - id: ["browserTimings/networkDuration", "browserTimings/sendDuration", "browserTimings/receiveDuration", "browserTimings/processingDuration", "browserTimings/totalDuration"] + segment: ["browserTiming/urlHost", "browserTiming/urlPath"] + aggregation: ["avg"] + interval: "P5M" + top: 5 + - id: ["exceptions/count", "exceptions/browser", "exceptions/server"] + segment: ["exception/type", "cloud/roleName"] + aggregation: ["sum"] + interval: "P5M" + - id: ["performanceCounters/memoryAvailableBytes", "performanceCounters/processCpuPercentageTotal", "performanceCounters/processCpuPercentage", "performanceCounters/processIOBytesPerSecond", + "performanceCounters/processPrivateBytes"] + aggregation: ["avg"] + segment: ["cloud/roleName", "cloud/roleInstance"] + interval: "P5M" diff --git a/x-pack/metricbeat/module/azure/client.go b/x-pack/metricbeat/module/azure/client.go index e488fab98b6..dd48f962b59 100644 --- a/x-pack/metricbeat/module/azure/client.go +++ b/x-pack/metricbeat/module/azure/client.go @@ -65,14 +65,14 @@ func (client *Client) InitResources(fn mapResourceMetrics) error { err = errors.Wrap(err, "failed to retrieve resources") return err } - if len(resourceList.Values()) == 0 { + if len(resourceList) == 0 { err = errors.Errorf("failed to retrieve resources: No resources returned using the configuration options resource ID %s, resource group %s, resource type %s, resource query %s", resource.Id, resource.Group, resource.Type, resource.Query) client.Log.Error(err) continue } //map resources to the client - for _, resource := range resourceList.Values() { + for _, resource := range resourceList { if !containsResource(*resource.ID, client.Resources) { client.Resources = append(client.Resources, Resource{ Id: *resource.ID, @@ -84,7 +84,7 @@ func (client *Client) InitResources(fn mapResourceMetrics) error { Subscription: client.Config.SubscriptionId}) } } - resourceMetrics, err := fn(client, resourceList.Values(), resource) + resourceMetrics, err := fn(client, resourceList, resource) if err != nil { return err } diff --git a/x-pack/metricbeat/module/azure/client_test.go b/x-pack/metricbeat/module/azure/client_test.go index 47b88f99cce..6b0df97a370 100644 --- a/x-pack/metricbeat/module/azure/client_test.go +++ b/x-pack/metricbeat/module/azure/client_test.go @@ -50,7 +50,7 @@ func TestInitResources(t *testing.T) { client := NewMockClient() client.Config = resourceQueryConfig m := &MockService{} - m.On("GetResourceDefinitions", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(resources.ListResultPage{}, errors.New("invalid resource query")) + m.On("GetResourceDefinitions", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return([]resources.GenericResource{}, errors.New("invalid resource query")) client.AzureMonitorService = m mr := MockReporterV2{} mr.On("Error", mock.Anything).Return(true) diff --git a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json index 1da5cfb63ab..c38407c76f1 100644 --- a/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm/_meta/data.json @@ -2,62 +2,86 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm": { + "disk_read_bytes": { + "total": 0 + }, "disk_read_operations_per_sec": { - "avg": 3.3875 + "avg": 0 + }, + "disk_write_bytes": { + "total": 2969709.4 }, "disk_write_operations_per_sec": { - "avg": 0.6705 + "avg": 0.7809999999999999 }, "inbound_flows": { - "avg": 28.4 + "avg": 10 }, "inbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 + }, + "network_in": { + "total": 1478232 + }, + "network_in_total": { + "total": 1569665 + }, + "network_out": { + "total": 793344 + }, + "network_out_total": { + "total": 1074624 + }, + "os_disk_bandwidth_consumed_percentage": { + "avg": 0 + }, + "os_disk_iops_consumed_percentage": { + "avg": 0 }, "os_disk_queue_depth": { - "avg": 0.00125 + "avg": 0.002 }, "os_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "os_per_disk_qd": { - "avg": 0.00125 + "avg": 0.002 }, "os_per_disk_read_bytes_per_sec": { - "avg": 602589.1825 + "avg": 0 }, "os_per_disk_read_operations_per_sec": { - "avg": 5.28375 + "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 14137.59375 + "avg": 9899.025 }, "os_per_disk_write_operations_per_sec": { - "avg": 1.46875 + "avg": 1.5619999999999998 }, "outbound_flows": { - "avg": 28.4 + "avg": 10 }, "outbound_flows_maximum_creation_rate": { - "avg": 10.4 + "avg": 10.6 }, "per_disk_qd": { - "avg": 0.0025 + "avg": 0 }, "per_disk_read_bytes_per_sec": { - "avg": 51985.035 + "avg": 0 }, "per_disk_read_operations_per_sec": { - "avg": 2.92875 + "avg": 0 }, "per_disk_write_bytes_per_sec": { "avg": 0 @@ -66,26 +90,41 @@ "avg": 0 }, "percentage_cpu": { - "avg": 9.747 + "avg": 1.8719999999999999 + }, + "vm_cached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_cached_iops_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_bandwidth_consumed_percentage": { + "avg": 0 + }, + "vm_uncached_iops_consumed_percentage": { + "avg": 0 } }, "namespace": "Microsoft.Compute/virtualMachines", "resource": { "group": "obs-infrastructure", - "id": "/subscriptions/70bd6e64-4b1e-4835-8896-db77b8eef364/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/obslinux", - "name": "obslinux", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachines/testaz", + "name": "testaz", "type": "Microsoft.Compute/virtualMachines" }, - "subscription_id": "70bd6e64-4b1e-4835-8896-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz" }, "machine": { - "type": "Basic_A0" + "type": "Standard_A1_v2" }, "provider": "azure", "region": "westeurope" @@ -97,10 +136,28 @@ }, "host": { "cpu": { - "pct": 0.09747 + "pct": 0.01872 }, - "id": "d5d9444a-1964-4d23-9c62-5463ecb16fe0", - "name": "obslinux" + "disk": { + "read": { + "bytes": 0 + }, + "write": { + "bytes": 2969709.4 + } + }, + "id": "490fe4cf-2b33-4ead-a016-7e614c2f48ad", + "name": "testaz", + "network": { + "in": { + "bytes": 1569665, + "packets": 1478232 + }, + "out": { + "bytes": 1074624, + "packets": 793344 + } + } }, "metricset": { "name": "compute_vm", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json index e8f59859d8b..2d99d8ff6b0 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/_meta/data.json @@ -2,6 +2,12 @@ "@timestamp": "2017-10-12T08:05:34.853Z", "azure": { "compute_vm_scaleset": { + "cpu_credits_consumed": { + "avg": 0.006999999999999999 + }, + "cpu_credits_remaining": { + "avg": 84.72 + }, "os_per_disk_qd": { "avg": 0 }, @@ -12,35 +18,34 @@ "avg": 0 }, "os_per_disk_write_bytes_per_sec": { - "avg": 1872.1200000000001 + "avg": 3846.531 }, "os_per_disk_write_operations_per_sec": { - "avg": 0.296 + "avg": 0.5519999999999999 } }, "namespace": "Microsoft.Compute/virtualMachineScaleSets", "resource": { - "group": "testgroup", - "id": "/subscriptions/70bd6e23-e3er3-4835-6785-db77b8eef364/resourceGroups/testgroup/providers/Microsoft.Compute/virtualMachineScaleSets/vmscaleset", - "name": "vmscaleset", - "tags": { - "environment": "staging", - "role": "allocator" - }, + "group": "obs-infrastructure", + "id": "/subscriptions/fd675b6f-b5e5-426e-ac45-d1f876d0ffa6/resourceGroups/obs-infrastructure/providers/Microsoft.Compute/virtualMachineScaleSets/obslinuxvmss", + "name": "obslinuxvmss", "type": "Microsoft.Compute/virtualMachineScaleSets" }, - "subscription_id": "70bd6e23-e3er3-4835-6785-db77b8eef364", + "subscription_id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6", "timegrain": "PT5M" }, "cloud": { + "account": { + "id": "fd675b6f-b5e5-426e-ac45-d1f876d0ffa6" + }, "instance": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "machine": { - "type": "Standard_D4s_v3" + "type": "Standard_B1ls" }, "provider": "azure", - "region": "eastus2" + "region": "westeurope" }, "event": { "dataset": "azure.compute_vm_scaleset", @@ -48,7 +53,7 @@ "module": "azure" }, "host": { - "name": "vmscaleset" + "name": "obslinuxvmss" }, "metricset": { "name": "compute_vm_scaleset", diff --git a/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml index 9369a36b79e..37b1293a2e5 100644 --- a/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml +++ b/x-pack/metricbeat/module/azure/compute_vm_scaleset/manifest.yml @@ -12,8 +12,10 @@ input: - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" timegrain: "PT5M" + ignore_unsupported: true - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "SlotId" @@ -24,6 +26,7 @@ input: "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "VMName" @@ -40,9 +43,11 @@ input: metrics: - name: ["CPU Credits Remaining", "CPU Credits Consumed", "OS Per Disk Read Bytes/sec", "OS Per Disk Write Bytes/sec", "OS Per Disk Read Operations/Sec", "OS Per Disk Write Operations/Sec", "OS Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" - name: ["Per Disk Read Bytes/sec", "Per Disk Write Bytes/sec", "Per Disk Read Operations/Sec", "Per Disk Write Operations/Sec", "Per Disk QD"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "SlotId" @@ -53,6 +58,7 @@ input: "Premium Data Disk Cache Read Hit", "Outbound Flows Maximum Creation Rate", "Inbound Flows Maximum Creation Rate", "Outbound Flows", "Inbound Flows", "OS Disk IOPS Consumed Percentage", "OS Disk Bandwidth Consumed Percentage", "OS Disk Queue Depth", "OS Disk Write Operations/Sec", "OS Disk Read Operations/Sec", "OS Disk Write Bytes/sec", "OS Disk Read Bytes/sec", "Data Disk IOPS Consumed Percentage"] namespace: "Microsoft.Compute/virtualMachineScaleSets" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "VMName" diff --git a/x-pack/metricbeat/module/azure/config.go b/x-pack/metricbeat/module/azure/config.go index 63bb5450b57..00f1af56126 100644 --- a/x-pack/metricbeat/module/azure/config.go +++ b/x-pack/metricbeat/module/azure/config.go @@ -41,7 +41,7 @@ type MetricConfig struct { Dimensions []DimensionConfig `config:"dimensions"` Timegrain string `config:"timegrain"` // namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if namespace is unsupported - // info messages will be logged instead + // info messages will be logged instead. Same situation with metrics, some are being removed from the API, we would like to make sure that does not affect the module IgnoreUnsupported bool `config:"ignore_unsupported"` } diff --git a/x-pack/metricbeat/module/azure/container_instance/manifest.yml b/x-pack/metricbeat/module/azure/container_instance/manifest.yml index a0e6cd5ec27..c767e072a42 100644 --- a/x-pack/metricbeat/module/azure/container_instance/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_instance/manifest.yml @@ -10,21 +10,25 @@ input: metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" - resource_id: "" metrics: - name: ["CpuUsage", "MemoryUsage"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "containerName" value: "*" - name: ["NetworkBytesReceivedPerSecond", "NetworkBytesTransmittedPerSecond"] namespace: "Microsoft.ContainerInstance/containerGroups" + ignore_unsupported: true timegrain: "PT5M" diff --git a/x-pack/metricbeat/module/azure/container_service/manifest.yml b/x-pack/metricbeat/module/azure/container_service/manifest.yml index 1384a268868..b7cf7639d42 100644 --- a/x-pack/metricbeat/module/azure/container_service/manifest.yml +++ b/x-pack/metricbeat/module/azure/container_service/manifest.yml @@ -10,6 +10,7 @@ input: metrics: - name: ["kube_node_status_condition"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "node" @@ -20,9 +21,11 @@ input: value: "*" - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" - name: ["kube_pod_status_ready", "kube_pod_status_phase"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "pod" @@ -31,6 +34,7 @@ input: metrics: - name: ["kube_node_status_condition"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "node" @@ -41,9 +45,11 @@ input: value: "*" - name: ["kube_node_status_allocatable_cpu_cores", "kube_node_status_allocatable_memory_bytes"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" - name: ["kube_pod_status_ready", "kube_pod_status_phase"] namespace: "Microsoft.ContainerService/managedClusters" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "pod" diff --git a/x-pack/metricbeat/module/azure/data.go b/x-pack/metricbeat/module/azure/data.go index bf77f657416..eb7f1433142 100644 --- a/x-pack/metricbeat/module/azure/data.go +++ b/x-pack/metricbeat/module/azure/data.go @@ -82,7 +82,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupDimValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupDimValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } } @@ -90,7 +90,7 @@ func EventsMapping(metrics []Metric, client *Client, report mb.ReporterV2) error event, metricList = createEvent(timestamp, defaultMetric, resource, groupTimeValues) if client.Config.AddCloudMetadata { vm = client.GetVMForMetaData(&resource, groupTimeValues) - addCloudVMMetadata(&event, vm) + addCloudVMMetadata(&event, vm, resource.Subscription) } } if client.Config.DefaultResourceType == "" { @@ -120,7 +120,7 @@ func managePropertyName(metric string) string { // create an object in case of ":" resultMetricName = strings.Replace(resultMetricName, "_-_", "_", -1) // replace uppercases with underscores - resultMetricName = replaceUpperCase(resultMetricName) + resultMetricName = ReplaceUpperCase(resultMetricName) // avoid cases as this "logicaldisk_avg._disk_sec_per_transfer" obj := strings.Split(resultMetricName, ".") @@ -134,8 +134,8 @@ func managePropertyName(metric string) string { return resultMetricName } -// replaceUpperCase func will replace upper case with '_' -func replaceUpperCase(src string) string { +// ReplaceUpperCase func will replace upper case with '_' +func ReplaceUpperCase(src string) string { replaceUpperCaseRegexp := regexp.MustCompile(replaceUpperCaseRegex) return replaceUpperCaseRegexp.ReplaceAllStringFunc(src, func(str string) string { var newStr string diff --git a/x-pack/metricbeat/module/azure/data_test.go b/x-pack/metricbeat/module/azure/data_test.go index cdfad1965f8..cca7edc0233 100644 --- a/x-pack/metricbeat/module/azure/data_test.go +++ b/x-pack/metricbeat/module/azure/data_test.go @@ -47,10 +47,10 @@ func TestGetDimensionValue(t *testing.T) { } func TestReplaceUpperCase(t *testing.T) { - result := replaceUpperCase("TestReplaceUpper_Case") + result := ReplaceUpperCase("TestReplaceUpper_Case") assert.Equal(t, result, "Test_replace_upper_Case") // should not split on acronyms - result = replaceUpperCase("CPU_Percentage") + result = ReplaceUpperCase("CPU_Percentage") assert.Equal(t, result, "CPU_Percentage") } diff --git a/x-pack/metricbeat/module/azure/database_account/manifest.yml b/x-pack/metricbeat/module/azure/database_account/manifest.yml index 39086f6ff66..3436008db7e 100644 --- a/x-pack/metricbeat/module/azure/database_account/manifest.yml +++ b/x-pack/metricbeat/module/azure/database_account/manifest.yml @@ -14,12 +14,14 @@ input: - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" value: "*" - name: ["TotalRequestUnits", "TotalRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" @@ -28,6 +30,7 @@ input: value: "*" - name: ["CassandraRequestCharges", "CassandraRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT1M" dimensions: - name: "DatabaseName" @@ -38,6 +41,7 @@ input: "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete", "TableTableThroughputUpdate","TableTableUpdate"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true dimensions: - name: "ResourceName" value: "*" @@ -49,12 +53,14 @@ input: - name: ["AvailableStorage", "DataUsage","DocumentCount", "DocumentQuota", "IndexUsage", "MetadataRequests", "MongoRequestCharge", "MongoRequests", "MongoRequestsCount", "MongoRequestsInsert", "MongoRequestsDelete", "MongoRequestsQuery", "MongoRequestsUpdate","ProvisionedThroughput", "NormalizedRUConsumption"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" value: "*" - name: ["TotalRequestUnits", "TotalRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT5M" dimensions: - name: "DatabaseName" @@ -63,6 +69,7 @@ input: value: "*" - name: ["CassandraRequestCharges", "CassandraRequests"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true timegrain: "PT1M" dimensions: - name: "DatabaseName" @@ -73,6 +80,7 @@ input: "SqlContainerDelete", "SqlContainerThroughputUpdate", "SqlContainerUpdate", "SqlDatabaseDelete", "SqlDatabaseThroughputUpdate", "SqlDatabaseUpdate", "TableTableDelete", "TableTableThroughputUpdate","TableTableUpdate"] namespace: "Microsoft.DocumentDb/databaseAccounts" + ignore_unsupported: true dimensions: - name: "ResourceName" value: "*" diff --git a/x-pack/metricbeat/module/azure/fields.go b/x-pack/metricbeat/module/azure/fields.go index 4c0ad95ad98..6a308a5d2ae 100644 --- a/x-pack/metricbeat/module/azure/fields.go +++ b/x-pack/metricbeat/module/azure/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAzure returns asset data. // This is the base64 encoded gzipped contents of module/azure. func AssetAzure() string { - return "eJzUV8tu2zoQ3fsrBl4GSD7AiwsEt5suuuteGJNjhY1EEuQorfv1hR6kKVHyo1aKxIsAEcnzEM8MqUd4peMO8HfjaAPAiivawfa5/X+7AZDkhVOWldE7+G8DAP1cqI1sqnaJo4rQ0w5K3AAcFFXS77qJj6CxphN4++Ojbac609jhyQzDGCaFYlVT6VDpOBIgX+n40ziZPJ8F7n/fXwieexvETokZ3MDoyJvGCcoIUw9X0AUc8JaEOihKpU7tjiwfLY0Glh1fkBGktMvBHIATWbPUU4srcMfXkGNHw1j6p4dZWrP/QYInQ/3D4pywZEpRo7VKl8P87cP2NhN9bKKNTmwWmvavtziTmptjGqHAU0WCk9wENt/sI0Sh5P2cKSB8/ZIRSlWT9sro8T4t7NGF/bl2b85oHpVyIi4T3k/xTw836z5UBhcG/1b1t14MOOLGaZK5XLS2UNqr8oX9xf4T+/CeGK9TgNZWSmC3zTM857pSsnQcObi/R6SykvSl9J7RcSGR51vjzMAVvB1ovjZwkpZrM5KWy3zzYYUbmuE0tLBqK2wNDBqz5O5VVSldvkdoB2hALaHxWBJIYlTVtckVjXOkxXHdzM6iBkrriPFXIYyf7sbyPl3g/H+KFXszWXRck+aifbCuyxM4ZOAns0Y2WS7vJB5A88tQYO2SUHQVvGJ99vm60Bd6atLzXfAe4rPtYaiDwpJTRq7ehEOZ9fAwgY/HgBCmeZeoxTLvGZbzhoIbrFYtru4Q6mAhgw20B+NIoOfViQPwMnUfuJWPoqGVpmtjxzS1bZiKt3pyFo0//cYq/vl16qTyjIHCC6zIE38OJ1FuHJ/xphmVJtfeFBm1oI9tbVALQe0ZQ45K5dkdP4ehoPaMIU/uTX2WDRrE5h9/yLhHT8XQmT+ym6A1nCL5B6HRio27/p5aXnlLzYGnd9LlkvZsHJYfOiaDxPhe/wQAAP//ZxBMvQ==" + return "eJzkmU1v47YTxu/5FIMcA2w+gA9/IP+2hz0ULfpyJsbUWGEjkVxy6Kz76QuKkixLsizbspugOSywenme30MOJyLzBd5otwL8Ozh6AGDFBa3g8SX+//EBICMvnbKsjF7B/x4AID0LpclCEV9xVBB6WkGODwAbRUXmV9WDX0BjSXvx+MM7Gx91Jtj6yojDoUxXilVJuUOl2zuN5Bvt3o3LOtdHhdPPH68ELykGsVNyRLdxdORNcJIGht0MM+waHfCWpNoo6qL24x5E3lk6uHE88QmMBiW+DmYD3MEate5HXMC7HYahdhsYc//8NGpr1n+R5N6tdFFMgXUeESVaq3ReP//49HheiFQ2bYwKdlA08V9vcaRqzi7TVgo8FSS5UzeNmw/rVkKo7HrPriB8/XFgiNYWSuJifh29MbtMlaS9MvqwLI6UxIlymFsKE8gHnaMDNwBPj/jnp7O5N4XBIzcvpf45wYAjDk5TNsRFa4XSXuWv7E+2u7btr4lxHkF3mkd8ppqgZ3QsMuTxVjhyY0YvqkSH7zaepLOlHUlnx/3GqwXOaH79qoFFW18MUDOOlo7nw2Q3qZu+yX+9aBx9C+TZC2mC5mcfylHnsco4Yf1bUoZKeNp7g6qg7BbmSXmCIXhyTfig1bcwPvIXIPzpyU0Ye/JV17+J9+9J/GRuDPxKmuPioGxhhpeudvKb4Fk78+7JCVal0rkXmvjduDeRBVct3Gfc5kuR/T95Qe0FtRc0XrMAfbVS70Dnq/V7DpojSWpL4g5wtdV5fNYZGQtU53cZwL3deZhsGIu7EFZO03D0XVKld4NO/VOjPbE+O/71ON2QAJCb2YCCtlScQvLktjcnSiYTQJbcxrgStaQ0SbHFllQatxO4RVXguiCx3jH5Javp170tNLaQbKG1hcp2Pna9aIR1aot8L+raFWrXS6GlDcKSk6QZc0oL+W7s0gbYm6e1fW2Efwn+bGxlqhmL5MKTNDq7G/nXX1K1RD5I3oNdxloVhdL5LfYYtTSgjt86ceIzYlTF3A2qDM6RlrvRsbr4tGxUtZ1IR4zfhTS+Pw8Xz9APfa32CIYsOi5Js4gXlk25F4eB+D6sycJg93ulcS06PGLdf2XH/lNt+Rbc0KX6OrGRTNak+7GuN57cT9brIHYAZbLD0z24fsSbZZbkoSffnibIqj3coNTaZZ4cjtcbSg5YLLq4qpPOShYGso3txjiS6Hlx40b4uHUquIXPLupW2n237ZimtIFJbMveidfhH5QOKe5+arqnnAggvMSCPPHnSNLitvdHsmlGpckJpT3H39gfO1pNCw3tRCBHufLsdp8jUEM7ESjua9RnmaAadhAnQ8Y1ehJ1Z/7IaRrW5rfIIExptGLj5n+n5jO/UofC/W/S40vas3FxK/KBB7ZGbMf1nwAAAP//oQy/EA==" } diff --git a/x-pack/metricbeat/module/azure/mock_service.go b/x-pack/metricbeat/module/azure/mock_service.go index f6f54c300e0..601f1de5b45 100644 --- a/x-pack/metricbeat/module/azure/mock_service.go +++ b/x-pack/metricbeat/module/azure/mock_service.go @@ -24,9 +24,9 @@ func (client *MockService) GetResourceDefinitionById(id string) (resources.Gener } // GetResourceDefinitions is a mock function for the azure service -func (client *MockService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { +func (client *MockService) GetResourceDefinitions(id []string, group []string, rType string, query string) ([]resources.GenericResource, error) { args := client.Called(id, group, rType, query) - return args.Get(0).(resources.ListResultPage), args.Error(1) + return args.Get(0).([]resources.GenericResource), args.Error(1) } // GetMetricDefinitions is a mock function for the azure service diff --git a/x-pack/metricbeat/module/azure/module.yml b/x-pack/metricbeat/module/azure/module.yml index a51b202612b..4170154e246 100644 --- a/x-pack/metricbeat/module/azure/module.yml +++ b/x-pack/metricbeat/module/azure/module.yml @@ -4,5 +4,6 @@ metricsets: - container_instance - container_service - database_account + - app_state - compute_vm - compute_vm_scaleset diff --git a/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc b/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc index 19254f4ada9..9957b6a2ddd 100644 --- a/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/azure/monitor/_meta/docs.asciidoc @@ -50,6 +50,9 @@ Metrics with dimensions are exported as flattened single dimensional metrics, ag `name`:: Dimension key `value`:: Dimension value. (Users can select * to return metric values for each dimension) +`ignore_unsupported`:: (_bool_) Namespaces can be unsupported by some resources and supported in some, this configuration option makes sure no error messages are returned if the namespace is unsupported. +The same will go for the metrics configured, some can be removed from Azure Monitor and it should not affect the state of the module. + Users can select the options to retrieve all metrics from a specific namespace using the following: ["source","yaml"] diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper.go b/x-pack/metricbeat/module/azure/monitor/client_helper.go index 82875f46de5..403b971496f 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper.go @@ -78,20 +78,20 @@ func filterMetricNames(resourceId string, metricConfig azure.MetricConfig, metri } } else { // verify if configured metric names are valid, return log error event for the invalid ones, map only the valid metric names - supportedMetricNames, unsupportedMetricNames = filterSConfiguredMetrics(metricConfig.Name, metricDefinitions) - if len(unsupportedMetricNames) > 0 { + supportedMetricNames, unsupportedMetricNames = filterConfiguredMetrics(metricConfig.Name, metricDefinitions) + if len(unsupportedMetricNames) > 0 && !metricConfig.IgnoreUnsupported { return nil, errors.Errorf("the metric names configured %s are not supported for the resource %s and namespace %s", strings.Join(unsupportedMetricNames, ","), resourceId, metricConfig.Namespace) } } - if len(supportedMetricNames) == 0 { + if len(supportedMetricNames) == 0 && !metricConfig.IgnoreUnsupported { return nil, errors.Errorf("the metric names configured : %s are not supported for the resource %s and namespace %s ", strings.Join(metricConfig.Name, ","), resourceId, metricConfig.Namespace) } return supportedMetricNames, nil } -// filterSConfiguredMetrics will filter out any unsupported metrics based on the namespace selected -func filterSConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) { +// filterConfiguredMetrics will filter out any unsupported metrics based on the namespace selected +func filterConfiguredMetrics(selectedRange []string, allRange []insights.MetricDefinition) ([]string, []string) { var inRange []string var notInRange []string var allMetrics string diff --git a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go index a15ee0089b9..bcbad0f4c26 100644 --- a/x-pack/metricbeat/module/azure/monitor/client_helper_test.go +++ b/x-pack/metricbeat/module/azure/monitor/client_helper_test.go @@ -108,7 +108,7 @@ func TestMapMetric(t *testing.T) { func TestFilterSConfiguredMetrics(t *testing.T) { selectedRange := []string{"TotalRequests", "Capacity", "CPUUsage"} - intersection, difference := filterSConfiguredMetrics(selectedRange, *MockMetricDefinitions()) + intersection, difference := filterConfiguredMetrics(selectedRange, *MockMetricDefinitions()) assert.Equal(t, intersection, []string{"TotalRequests", "Capacity"}) assert.Equal(t, difference, []string{"CPUUsage"}) } diff --git a/x-pack/metricbeat/module/azure/monitor_service.go b/x-pack/metricbeat/module/azure/monitor_service.go index 053da3db05b..c3ed4e2fa43 100644 --- a/x-pack/metricbeat/module/azure/monitor_service.go +++ b/x-pack/metricbeat/module/azure/monitor_service.go @@ -55,16 +55,24 @@ func NewService(clientId string, clientSecret string, tenantId string, subscript } // GetResourceDefinitions will retrieve the azure resources based on the options entered -func (service MonitorService) GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) { +func (service MonitorService) GetResourceDefinitions(id []string, group []string, rType string, query string) ([]resources.GenericResource, error) { var resourceQuery string + var resourceList []resources.GenericResource if len(id) > 0 { - var filterList []string - // listing resourceID conditions does not seem to work with the API but querying by name or resource types will work + // listing multiple resourceId conditions does not seem to work with the API, extracting the name and resource type does not work as the position of the `resourceType` can move if a parent resource is involved, filtering by resource name and resource group (if extracted) is also not possible as + // different types of resources can contain the same name. for _, id := range id { - filterList = append(filterList, fmt.Sprintf("name eq '%s'", getResourceNameFromId(id))) + resource, err := service.resourceClient.List(service.context, fmt.Sprintf("resourceId eq '%s'", id), "", nil) + if err != nil { + return nil, err + } + if len(resource.Values()) > 0 { + resourceList = append(resourceList, resource.Values()...) + } } - resourceQuery = fmt.Sprintf("(%s) AND resourceType eq '%s'", strings.Join(filterList, " OR "), getResourceTypeFromId(id[0])) - } else if len(group) > 0 { + return resourceList, nil + } + if len(group) > 0 { var filterList []string for _, gr := range group { filterList = append(filterList, fmt.Sprintf("resourceGroup eq '%s'", gr)) @@ -76,7 +84,11 @@ func (service MonitorService) GetResourceDefinitions(id []string, group []string } else if query != "" { resourceQuery = query } - return service.resourceClient.List(service.context, resourceQuery, "", nil) + result, err := service.resourceClient.List(service.context, resourceQuery, "", nil) + if err == nil { + resourceList = result.Values() + } + return resourceList, err } // GetResourceDefinitionById will retrieve the azure resource based on the resource Id diff --git a/x-pack/metricbeat/module/azure/service_interface.go b/x-pack/metricbeat/module/azure/service_interface.go index e8985a7eedd..30430d03100 100644 --- a/x-pack/metricbeat/module/azure/service_interface.go +++ b/x-pack/metricbeat/module/azure/service_interface.go @@ -12,7 +12,7 @@ import ( // Service interface for the azure monitor service and mock for testing type Service interface { GetResourceDefinitionById(id string) (resources.GenericResource, error) - GetResourceDefinitions(id []string, group []string, rType string, query string) (resources.ListResultPage, error) + GetResourceDefinitions(id []string, group []string, rType string, query string) ([]resources.GenericResource, error) GetMetricDefinitions(resourceId string, namespace string) (insights.MetricDefinitionCollection, error) GetMetricNamespaces(resourceId string) (insights.MetricNamespaceCollection, error) GetMetricValues(resourceId string, namespace string, timegrain string, timespan string, metricNames []string, aggregations string, filter string) ([]insights.Metric, string, error) diff --git a/x-pack/metricbeat/module/azure/test/integration.go b/x-pack/metricbeat/module/azure/test/integration.go index 7187e3b93f7..39e8402110c 100644 --- a/x-pack/metricbeat/module/azure/test/integration.go +++ b/x-pack/metricbeat/module/azure/test/integration.go @@ -40,3 +40,23 @@ func GetConfig(t *testing.T, metricSetName string) map[string]interface{} { "subscription_id": subId, } } + +// GetConfigForInsights function gets azure credentials for integration tests. +func GetConfigForInsights(t *testing.T, metricSetName string) map[string]interface{} { + t.Helper() + applicationId, ok := os.LookupEnv("AZURE_APPLICATION_ID") + if !ok { + t.Fatal("Could not find var AZURE_APPLICATION_ID") + } + apiKey, ok := os.LookupEnv("AZURE_API_KEY") + if !ok { + t.Fatal("Could not find var AZURE_API_KEY") + } + return map[string]interface{}{ + "module": "azure", + "period": "300s", + "metricsets": []string{metricSetName}, + "application_id": applicationId, + "api_key": apiKey, + } +} diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml index be15db23b65..6299cfc8116 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.reference.yml @@ -10,4 +10,5 @@ rlp_address: '${CLOUDFOUNDRY_RLP_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat version: v1 diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml index a2803b06d40..5ea86f3e8de 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/config.yml @@ -7,3 +7,4 @@ api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat diff --git a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc index 4d908802358..752b2aaea0c 100644 --- a/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/cloudfoundry/_meta/docs.asciidoc @@ -107,7 +107,7 @@ Client Secret to authenticate with Cloud Foundry. Default: "". === `shard_id` Shard ID for connection to the RLP Gateway. Use the same ID across multiple {beatname_lc} to shard the load of events -from the RLP Gateway. Default: "(generated UUID)". +from the RLP Gateway. [float] ==== `version` diff --git a/x-pack/metricbeat/module/istio/_meta/config.reference.yml b/x-pack/metricbeat/module/istio/_meta/config.reference.yml index c3e940f80e5..146728fdfcd 100644 --- a/x-pack/metricbeat/module/istio/_meta/config.reference.yml +++ b/x-pack/metricbeat/module/istio/_meta/config.reference.yml @@ -32,3 +32,10 @@ period: 10s # use istio-pilot.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset hosts: ["localhost:15014"] + +# Istio istiod to monitor the Istio Daemon for versions after 1.5 of Istio. +- module: istio + metricsets: ['istiod'] + period: 10s + # use istiod.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset + hosts: ['localhost:15014'] diff --git a/x-pack/metricbeat/module/istio/_meta/config.yml b/x-pack/metricbeat/module/istio/_meta/config.yml index c3e940f80e5..146728fdfcd 100644 --- a/x-pack/metricbeat/module/istio/_meta/config.yml +++ b/x-pack/metricbeat/module/istio/_meta/config.yml @@ -32,3 +32,10 @@ period: 10s # use istio-pilot.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset hosts: ["localhost:15014"] + +# Istio istiod to monitor the Istio Daemon for versions after 1.5 of Istio. +- module: istio + metricsets: ['istiod'] + period: 10s + # use istiod.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset + hosts: ['localhost:15014'] diff --git a/x-pack/metricbeat/module/istio/_meta/docs.asciidoc b/x-pack/metricbeat/module/istio/_meta/docs.asciidoc index 0a52294d1fc..cfba8ec7ce8 100644 --- a/x-pack/metricbeat/module/istio/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/istio/_meta/docs.asciidoc @@ -1,9 +1,22 @@ -This is the Istio module. The Istio module collects metrics from the +This is the Istio module. +This module is compatible with versions before `1.5` of Istio where microservices architecture is used. If using +versions priot to `1.5` then `mesh`, `mixer`, `pilot`, `galley`, `citadel` metricsets should be used. +wehre the Istio module collects metrics from the Istio https://istio.io/v1.4/docs/tasks/observability/metrics/querying-metrics/#about-the-prometheus-add-on[prometheus exporters endpoints]. +For versions after `1.5`, `istiod` metricset can be used which collects metrics directly from Istio Daemon. The default metricsets are `mesh`, `mixer`, `pilot`, `galley`, `citadel`. [float] === Compatibility -The Istio module is tested with Istio `1.4`. +The Istio module is tested with Istio `1.4` for `mesh`, `mixer`, `pilot`, `galley`, `citadel`. +The Istio module is tested with Istio `1.7` for `istiod`. + +[float] +=== Dashboard + +The Istio module includes a predefined dashboard with overview information about Istio Daemon. +This dashboard is only compatible with versions of Istio after `1.5` which should be monitored with `istiod` metricset. + +image::./images/metricbeat-istio-overview.png[] diff --git a/x-pack/metricbeat/module/istio/_meta/kibana/7/dashboard/Metricbeat-istio-overview.json b/x-pack/metricbeat/module/istio/_meta/kibana/7/dashboard/Metricbeat-istio-overview.json new file mode 100644 index 00000000000..72b995c2382 --- /dev/null +++ b/x-pack/metricbeat/module/istio/_meta/kibana/7/dashboard/Metricbeat-istio-overview.json @@ -0,0 +1,1762 @@ +{ + "objects": [ + { + "attributes": { + "description": "Overview of the Istiod Service status", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "title": "Pilot Proxy Queue Time" + }, + "gridData": { + "h": 9, + "i": "cd1bbc4f-95de-4156-a3ef-c091cf6402c0", + "w": 12, + "x": 0, + "y": 0 + }, + "panelIndex": "cd1bbc4f-95de-4156-a3ef-c091cf6402c0", + "panelRefName": "panel_0", + "title": "Pilot Proxy Queue Time", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot xds Push Time" + }, + "gridData": { + "h": 9, + "i": "06af11e6-e026-48db-a06b-b34b402b535b", + "w": 12, + "x": 12, + "y": 0 + }, + "panelIndex": "06af11e6-e026-48db-a06b-b34b402b535b", + "panelRefName": "panel_1", + "title": "Pilot xds Push Time", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot xds Pushes" + }, + "gridData": { + "h": 9, + "i": "d9a49bf0-f88b-4d4f-a1e2-74fbd482f77c", + "w": 11, + "x": 24, + "y": 0 + }, + "panelIndex": "d9a49bf0-f88b-4d4f-a1e2-74fbd482f77c", + "panelRefName": "panel_2", + "title": "Pilot xds Pushes", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot Inbound Updates" + }, + "gridData": { + "h": 9, + "i": "a8e47ef0-03db-419f-890f-0880d674682c", + "w": 13, + "x": 35, + "y": 0 + }, + "panelIndex": "a8e47ef0-03db-419f-890f-0880d674682c", + "panelRefName": "panel_3", + "title": "Pilot Inbound Updates", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Citadel Cert Issuane" + }, + "gridData": { + "h": 9, + "i": "e708abfa-5a95-483c-9bb2-4470ee913f3c", + "w": 12, + "x": 0, + "y": 9 + }, + "panelIndex": "e708abfa-5a95-483c-9bb2-4470ee913f3c", + "panelRefName": "panel_4", + "title": "Citadel Cert Issuane", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Galley Validation Failed" + }, + "gridData": { + "h": 9, + "i": "724f0f9e-2186-4ddd-859c-edb2649b8c0f", + "w": 12, + "x": 12, + "y": 9 + }, + "panelIndex": "724f0f9e-2186-4ddd-859c-edb2649b8c0f", + "panelRefName": "panel_5", + "title": "Galley Validation Failed", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pods witout IP", + "vis": null + }, + "gridData": { + "h": 9, + "i": "32eaa989-a4f9-4d31-97cb-684f31488aa8", + "w": 8, + "x": 24, + "y": 9 + }, + "panelIndex": "32eaa989-a4f9-4d31-97cb-684f31488aa8", + "panelRefName": "panel_6", + "title": "Pods witout IP", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot Virtual Services", + "vis": null + }, + "gridData": { + "h": 9, + "i": "6a8463fe-b7cb-4cd8-bf01-f7ca6a185178", + "w": 8, + "x": 32, + "y": 9 + }, + "panelIndex": "6a8463fe-b7cb-4cd8-bf01-f7ca6a185178", + "panelRefName": "panel_7", + "title": "Pilot Virtual Services", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot Services", + "vis": null + }, + "gridData": { + "h": 9, + "i": "51ecc2f8-3c3f-4a80-b4b6-b52db10e68ad", + "w": 8, + "x": 40, + "y": 9 + }, + "panelIndex": "51ecc2f8-3c3f-4a80-b4b6-b52db10e68ad", + "panelRefName": "panel_8", + "title": "Pilot Services", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot Conflict Inbound Listener", + "vis": null + }, + "gridData": { + "h": 9, + "i": "0a63d980-8d93-4ce1-b5a1-ab77e589ceec", + "w": 9, + "x": 0, + "y": 18 + }, + "panelIndex": "0a63d980-8d93-4ce1-b5a1-ab77e589ceec", + "panelRefName": "panel_9", + "title": "Pilot Conflict Inbound Listener", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot eds instances", + "vis": null + }, + "gridData": { + "h": 9, + "i": "9fbfca4c-37b5-4a1a-924e-49fc9ef2294c", + "w": 10, + "x": 9, + "y": 18 + }, + "panelIndex": "9fbfca4c-37b5-4a1a-924e-49fc9ef2294c", + "panelRefName": "panel_10", + "title": "Pilot eds instances", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot xds endpoints" + }, + "gridData": { + "h": 9, + "i": "d6b2845f-9582-4863-853e-ab753f3d763e", + "w": 14, + "x": 19, + "y": 18 + }, + "panelIndex": "d6b2845f-9582-4863-853e-ab753f3d763e", + "panelRefName": "panel_11", + "title": "Pilot xds endpoints", + "version": "7.8.0" + }, + { + "embeddableConfig": { + "title": "Pilot xds expired" + }, + "gridData": { + "h": 9, + "i": "3d0dec37-26c3-490b-a45f-48b6d1baa160", + "w": 15, + "x": 33, + "y": 18 + }, + "panelIndex": "3d0dec37-26c3-490b-a45f-48b6d1baa160", + "panelRefName": "panel_12", + "title": "Pilot xds expired", + "version": "7.8.0" + } + ], + "timeRestore": false, + "title": "[Metricbeat Istio] Overview", + "version": 1 + }, + "id": "d899d3f0-0883-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "dashboard": "7.3.0" + }, + "references": [ + { + "id": "dd1392f0-07d8-11eb-a3fd-1b45ec532bb3", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "b5b3abb0-087c-11eb-a3fd-1b45ec532bb3", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "f858c200-087e-11eb-a3fd-1b45ec532bb3", + "name": "panel_2", + "type": "visualization" + }, + { + "id": "aa997510-087d-11eb-a3fd-1b45ec532bb3", + "name": "panel_3", + "type": "visualization" + }, + { + "id": "506c8490-087f-11eb-a3fd-1b45ec532bb3", + "name": "panel_4", + "type": "visualization" + }, + { + "id": "98b01f00-087f-11eb-a3fd-1b45ec532bb3", + "name": "panel_5", + "type": "visualization" + }, + { + "id": "4275f710-0882-11eb-a3fd-1b45ec532bb3", + "name": "panel_6", + "type": "visualization" + }, + { + "id": "96bfe060-0882-11eb-a3fd-1b45ec532bb3", + "name": "panel_7", + "type": "visualization" + }, + { + "id": "6cfbe3f0-0882-11eb-a3fd-1b45ec532bb3", + "name": "panel_8", + "type": "visualization" + }, + { + "id": "d62a1e60-0881-11eb-a3fd-1b45ec532bb3", + "name": "panel_9", + "type": "visualization" + }, + { + "id": "12cdcce0-0882-11eb-a3fd-1b45ec532bb3", + "name": "panel_10", + "type": "visualization" + }, + { + "id": "e5f3e870-0882-11eb-a3fd-1b45ec532bb3", + "name": "panel_11", + "type": "visualization" + }, + { + "id": "0ed17c80-0883-11eb-a3fd-1b45ec532bb3", + "name": "panel_12", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2020-10-07T10:23:52.518Z", + "version": "WzQ0OTIsMV0=" + }, + { + "attributes": { + "description": "Time in seconds, a proxy is in the push queue before being dequeued.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Proxy Queue Time [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "queue_time", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_proxy_queue_time.histogram.values", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "03ef6580-0887-11eb-876a-9d8e5e94d2f5", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "071fe4f0-0887-11eb-876a-9d8e5e94d2f5", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "0b7164c0-0887-11eb-876a-9d8e5e94d2f5", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "90" + }, + { + "id": "0f611580-0887-11eb-876a-9d8e5e94d2f5", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "136f98e0-0887-11eb-876a-9d8e5e94d2f5", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "percentile" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot Proxy Queue Time [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "dd1392f0-07d8-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T10:23:11.367Z", + "version": "WzQ0ODMsMV0=" + }, + { + "attributes": { + "description": "Total time in seconds Pilot takes to push lds, rds, cds and eds.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot xds Push Time [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "filter": { + "language": "kuery", + "query": "prometheus.labels.type: \"rds\" OR prometheus.labels.type: \"lds\" OR prometheus.labels.type: \"cds\" OR prometheus.labels.type: \"eds\"" + }, + "formatter": "s,s,", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "pilot_xds_push_time", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_xds_push_time.histogram.values", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "percentile" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "prometheus.labels.type", + "terms_size": "20", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot xds Push Time [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "b5b3abb0-087c-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T10:23:48.176Z", + "version": "WzQ0ODgsMV0=" + }, + { + "attributes": { + "description": "Pilot build and send errors for lds, rds, cds and eds.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot xds Pushes [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "pilot_xds_pushes", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_xds_pushes.counter", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "max", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "prometheus.labels.type", + "terms_order_by": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "terms_size": "4", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot xds Pushes [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "f858c200-087e-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T09:24:59.040Z", + "version": "WzQzOTAsMV0=" + }, + { + "attributes": { + "description": "Total number of updates received by pilot.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Inbound Updates [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "pilot_inbound_updates", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_inbound_updates.counter", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "max", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "prometheus.labels.type", + "terms_order_by": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "terms_size": "10", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot Inbound Updates [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "aa997510-087d-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T09:21:43.250Z", + "version": "WzQzODcsMV0=" + }, + { + "attributes": { + "description": "The number of certificates issuances that have succeeded.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Citadel Cert Issuance [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "success_cert_issuance", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.citadel_server_success_cert_issuance_count.counter", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "max", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "terms_field": "prometheus.labels.type", + "terms_order_by": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "terms_size": "4", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Citadel Cert Issuance [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "506c8490-087f-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T10:15:51.172Z", + "version": "WzQ0NzYsMV0=" + }, + { + "attributes": { + "description": "Resource validation failed.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Galley Validation Failed [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "filter": { + "language": "kuery", + "query": "" + }, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "galley_validation_failed", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.galley_validation_failed.counter", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "max", + "unit": "" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "terms", + "stacked": "none", + "terms_field": "prometheus.labels.resource", + "terms_order_by": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "terms_size": "10", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Galley Validation Failed [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "98b01f00-087f-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T09:30:05.212Z", + "version": "WzQzOTksMV0=" + }, + { + "attributes": { + "description": "Pods not found in the endpoint table, possibly invalid.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Pods without IP [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Pilot No IP pods", + "field": "prometheus.pilot_no_ip.value" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "gauge": { + "alignment": "automatic", + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 50 + }, + { + "from": 50, + "to": 75 + }, + { + "from": 75, + "to": 100 + } + ], + "extendRange": true, + "gaugeColorMode": "Labels", + "gaugeStyle": "Full", + "gaugeType": "Arc", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "rgba(105,112,125,0.2)", + "labels": false, + "show": true + }, + "style": { + "bgColor": true, + "bgFill": "rgba(105,112,125,0.2)", + "bgMask": false, + "bgWidth": 0.9, + "fontSize": 60, + "mask": false, + "maskBars": 50, + "subText": "", + "width": 0.9 + }, + "type": "meter" + }, + "isDisplayWarning": false, + "type": "gauge" + }, + "title": "Pilot Pods without IP [Metricbeat Istio]", + "type": "gauge" + } + }, + "id": "4275f710-0882-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-07T09:48:31.873Z", + "version": "WzQ0MjcsMV0=" + }, + { + "attributes": { + "description": "Total virtual services known to pilot.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Virtual Services [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Pilot Virtual Services", + "field": "prometheus.pilot_virt_services.value" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "gauge": { + "alignment": "automatic", + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 50 + }, + { + "from": 50, + "to": 75 + }, + { + "from": 75, + "to": 100 + } + ], + "extendRange": true, + "gaugeColorMode": "Labels", + "gaugeStyle": "Full", + "gaugeType": "Arc", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "rgba(105,112,125,0.2)", + "labels": false, + "show": true + }, + "style": { + "bgColor": true, + "bgFill": "rgba(105,112,125,0.2)", + "bgMask": false, + "bgWidth": 0.9, + "fontSize": 60, + "mask": false, + "maskBars": 50, + "subText": "", + "width": 0.9 + }, + "type": "meter" + }, + "isDisplayWarning": false, + "type": "gauge" + }, + "title": "Pilot Virtual Services [Metricbeat Istio]", + "type": "gauge" + } + }, + "id": "96bfe060-0882-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-07T09:50:53.286Z", + "version": "WzQ0MzMsMV0=" + }, + { + "attributes": { + "description": "Total services known to pilot.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Services [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Pilot Services", + "field": "prometheus.pilot_services.value" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "gauge": { + "alignment": "automatic", + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 50 + }, + { + "from": 50, + "to": 75 + }, + { + "from": 75, + "to": 100 + } + ], + "extendRange": true, + "gaugeColorMode": "Labels", + "gaugeStyle": "Full", + "gaugeType": "Arc", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "rgba(105,112,125,0.2)", + "labels": false, + "show": true + }, + "style": { + "bgColor": true, + "bgFill": "rgba(105,112,125,0.2)", + "bgMask": false, + "bgWidth": 0.9, + "fontSize": 60, + "mask": false, + "maskBars": 50, + "subText": "", + "width": 0.9 + }, + "type": "meter" + }, + "isDisplayWarning": false, + "type": "gauge" + }, + "title": "Pilot Services [Metricbeat Istio]", + "type": "gauge" + } + }, + "id": "6cfbe3f0-0882-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-07T10:04:27.677Z", + "version": "WzQ0NTksMV0=" + }, + { + "attributes": { + "description": "Number of conflicting inbound listeners.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot Conflict Inbound Listener [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Pilot conflict inbound listener", + "field": "prometheus.pilot_conflict_inbound_listener.value" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "gauge": { + "alignment": "automatic", + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 50 + }, + { + "from": 50, + "to": 75 + }, + { + "from": 75, + "to": 100 + } + ], + "extendRange": true, + "gaugeColorMode": "Labels", + "gaugeStyle": "Full", + "gaugeType": "Arc", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "rgba(105,112,125,0.2)", + "labels": false, + "show": true + }, + "style": { + "bgColor": true, + "bgFill": "rgba(105,112,125,0.2)", + "bgMask": false, + "bgWidth": 0.9, + "fontSize": 60, + "mask": false, + "maskBars": 50, + "subText": "", + "width": 0.9 + }, + "type": "meter" + }, + "isDisplayWarning": false, + "type": "gauge" + }, + "title": "Pilot Conflict Inbound Listener [Metricbeat Istio]", + "type": "gauge" + } + }, + "id": "d62a1e60-0881-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-07T09:45:30.182Z", + "version": "WzQ0MjAsMV0=" + }, + { + "attributes": { + "description": "Number of clusters without instances.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot eds Instances [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Pilot eds instnaces", + "field": "prometheus.pilot_eds_no_instances.value" + }, + "schema": "metric", + "type": "avg" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "gauge": { + "alignment": "automatic", + "backStyle": "Full", + "colorSchema": "Green to Red", + "colorsRange": [ + { + "from": 0, + "to": 50 + }, + { + "from": 50, + "to": 75 + }, + { + "from": 75, + "to": 100 + } + ], + "extendRange": true, + "gaugeColorMode": "Labels", + "gaugeStyle": "Full", + "gaugeType": "Arc", + "invertColors": false, + "labels": { + "color": "black", + "show": true + }, + "orientation": "vertical", + "percentageMode": false, + "scale": { + "color": "rgba(105,112,125,0.2)", + "labels": false, + "show": true + }, + "style": { + "bgColor": true, + "bgFill": "rgba(105,112,125,0.2)", + "bgMask": false, + "bgWidth": 0.9, + "fontSize": 60, + "mask": false, + "maskBars": 50, + "subText": "", + "width": 0.9 + }, + "type": "meter" + }, + "isDisplayWarning": false, + "type": "gauge" + }, + "title": "Pilot eds Instances [Metricbeat Istio]", + "type": "gauge" + } + }, + "id": "12cdcce0-0882-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [ + { + "id": "metricbeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2020-10-07T09:47:11.918Z", + "version": "WzQ0MjQsMV0=" + }, + { + "attributes": { + "description": "Number of endpoints connected to this pilot using XDS.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot XDS endpoints [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "pilot_xds", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_xds.value", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot XDS endpoints [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "e5f3e870-0882-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T09:53:38.583Z", + "version": "WzQ0NDEsMV0=" + }, + { + "attributes": { + "description": "Total number of XDS requests with an expired nonce.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Pilot XDS expired [Metricbeat Istio]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [], + "params": { + "axis_formatter": "number", + "axis_min": 0, + "axis_position": "left", + "axis_scale": "normal", + "default_index_pattern": "metricbeat-*", + "default_timefield": "@timestamp", + "filter": { + "language": "kuery", + "query": "" + }, + "id": "7ccbe640-07d8-11eb-985d-2f490d4c2901", + "index_pattern": "metricbeat-*", + "interval": "auto", + "isModelInvalid": false, + "series": [ + { + "axis_position": "right", + "chart_type": "line", + "color": "#6092C0", + "fill": 0, + "formatter": "number", + "id": "7ccbe641-07d8-11eb-985d-2f490d4c2901", + "label": "pilot_xds_expired_nonce", + "line_width": 2, + "metrics": [ + { + "field": "prometheus.pilot_xds_expired_nonce.counter", + "id": "7ccbe642-07d8-11eb-985d-2f490d4c2901", + "percentiles": [ + { + "id": "88c0d000-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "50" + }, + { + "id": "95c750d0-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "25" + }, + { + "id": "9c5ec900-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "75" + }, + { + "id": "9f8e3700-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "95" + }, + { + "id": "a3581040-07d8-11eb-86d1-6521145f6524", + "mode": "line", + "percentile": "", + "shade": 0.2, + "value": "99" + } + ], + "type": "avg" + } + ], + "point_size": 0, + "separate_axis": 0, + "split_mode": "everything", + "stacked": "none", + "type": "timeseries", + "value_template": "{{value}}" + } + ], + "show_grid": 1, + "show_legend": 1, + "time_field": "@timestamp", + "type": "timeseries" + }, + "title": "Pilot XDS expired [Metricbeat Istio]", + "type": "metrics" + } + }, + "id": "0ed17c80-0883-11eb-a3fd-1b45ec532bb3", + "migrationVersion": { + "visualization": "7.8.0" + }, + "references": [], + "type": "visualization", + "updated_at": "2020-10-07T09:54:14.728Z", + "version": "WzQ0NDMsMV0=" + } + ], + "version": "7.8.0" +} diff --git a/x-pack/metricbeat/module/istio/fields.go b/x-pack/metricbeat/module/istio/fields.go index 40a9baed3f5..ccd41e55fe8 100644 --- a/x-pack/metricbeat/module/istio/fields.go +++ b/x-pack/metricbeat/module/istio/fields.go @@ -19,5 +19,5 @@ func init() { // AssetIstio returns asset data. // This is the base64 encoded gzipped contents of module/istio. func AssetIstio() string { - return "eJzUXE2T47bRvu+v6PLltd+a5V5Te0hVap2PrcSuza5TSeUiQ2BLhAUCNNCURv71KXyRFEVp9AFq1nPYnZFIPM8DNBqNRpNvYYP79yAsCf0GgARJfA/ffHR/f/MGoETLjWhIaPUe/vgGAMK18IMuW4lvAAxKZBbfwxKJvQFYCZSlfe8vfQuK1dg3735o3+B7WBvdNvGTCQz387O/62fgWhETyoIlRu4zboEqRrBDg2CQlbAyuoaPA5AhiSERLoiVKLvPp+icoeR+PkzQMSgZYQmkgSoMTOBDwAKLZis4DhoZd1n6GbMeMl+bhhc1UqXLg++Tgg3ud9qMvzujw/38VKFvGCYaPgA+1pALearlA2gHkhfX3Q565YfKoG2Qk9ieoDPJyyI3SAtnmEZLiWZht3zBOF9wg84QFhwNFVy3iiapS63W1/NWbb1E45i75sVKcEZoIWJC2aIzwKgAGPf44WuhVXFCitmiWRityZNe4HMjzH5hkWtV2kn2K6nZWNcF9FslnoFEjZZY3TyBUBBRnmBXoeomjCPjJcJOSAmeERbwbwSLBIKcSoVr5kdNKODM+vEUitAoJgGN0WZab2fNaIqKqVLi9HS6YYQ0MTkYo8+fPljgum4kutHRyttbgH4Cg2tmSonWumtty7n/1cCKCdkafJl9bdeFQY5iO6cEsGSQ1VCjtWyNztUFyENBl9G1mG8+jKiup7g6PFjur+JpiRma1ygixKkevMxuhVoXbtFRfF/Utli2fINU/P8kb738Bfm458OHi3scknOeWlmESAS+rYWUIs7q77px8at1xUpYIipgTSOd6xJavZW4RQlxIl47VFP9YNv60qFbaVMzeg9lazyZDPqj4Cewbe3+CJ8LtM5NDfvmJm2515PT9HuDrZGM4PbYODvSTErcPySgClCZ4in3b96wwrU4CCt0azj6P0IfghjKOsnJNix3nPX3dolGoYsVOoQx0Uk+WzT2eGLcySY2mghM+KYuWHexFZ+Ymncy6NtNJISyxNRLYZ/fjxSspQoVRQ9W1GirRkvhZvkssd6WSVEG7HeH2O+G2LBReqfchInThBEwaLRQ5JyPi7suF/VagvKIcd1S/IC2+qDVSoy55hfh8N71ePeRV0g7bTbO9ZdoSSjfM6aVDxiNHvvdGDubKFRbvV8JSWgeKmiIm03MmhHu2P6hQhJmNhFWlMjZY0cjYWYTsRWGWiZjbPBQLSPo+yR5D7gvGJERy5awZkqs0NL8igLyu2PkLIJCiP8AI4syEl4W8ik8eBj7DjAL/ccsHpH61avF+ZjLtMpdXTRGc7RWmwK3qGhhG6aKtHN7tQ1wIuA2Hju3t0XGKxCK61qoNXiqwCxUwpJeG1ZDoHnZfvBi7Q/d9F6iudsAp4szC869E75M0wW74vNqrGKNrTQtfIOvZLO9isAiZWe2GNIzkS6Wk4Z7o9IrLDSnhM4Ow3U3kv+9W5sUK/TffT3+0g2JE5045nSS53W/jq8c683mIM+Lnc1yj/XcaK6WGKG3qcW8UZY/9OvjqgZN+Ohfn//xAkHDCNf7QqsFr5haTyfn7qYnarQh+x0B/8+CVh88IlTMBs/GXTg1fU57RNj9ZRa/tgItx4Vx45XvXGOCPIOI1dOdAu1OHn3ecbFpl7iIy3w4BVuQJiZnJbrpM6IxTxuxV62U++48gqngjKeToy72DTPNH5xucF+0TckomwH/NcTOPRDscFlpvRme/8IU5hRHn5sqpGa5bGDzB9sT8q0n7/Ctwe8c0KWk8nbbaV5jnMSoRlu9GQPPcYLhgDKdXxylrIfcCZ9vOKBvjK6RKmztdOsJ+he9nES9OSE/AB633Xk3/LXNmQbx57Kp0eEBJFNxrBqjn/fnuLxuOPU5kOiX5BvDpwktDwuRPqf+T7ffGhZNiMgZ+kzw7NeUScscM7PiNyyWe8JXtJQv4je8dEN3TDvfDu4EnzHAGS4zjO2YzSHECWLh+Pz3N7jHvF9xdI/JfC3D22hDaPItdZ9jiyBKVC5+ixF/QurP5IPFw0cCYX3RG2kYnMiBWMWb3Pe++JXFqpnh4gVMlSm+nbqDS4GKhnecqBH0TRQ7bTYuqityl04IO+6RVEoRySdk2FWCVxDrLuN2KZC7mHj2+opp9gdFFiMZZ8k2RigumhMboGwcG0QDHVYiSoatVoInLgMD9AWa/qbDI3v3fWvxvCbWNPOqiR3MmgaWzIbSOveHZEuUt4zCDJUvx6xHlTAn+U0SHbiD15iZQ290vkdPEn2NmXgb7a9iTg4J3T0xh43NPjuHnX5+ik4Oz8sKXmO2nrekFynHJEBRaTsd6mThPSSZngtwiCnsEkuJbg/ndr3d+UMKPy7WML/XmdLhUK/j+GiHM8H6BQtJXd8YTZrruf1NRDkfd/7p08f+SrFyv29FieUTaKrQ7ITt7gRtXHCmYnFlumt6lLrIn+syX079WGVX5exwjqT6O4IPEhYag/6ZAa3k3jmov/3006d0njGtopdbWOStEbSPZRczRzzpKZ9Dpx+gzw9o3VLL5IKkDYtHiP/jcuEvYBvXW3XdqtSu14Z+QzHaRpxYlkhDq0IViAcZ3ZWy7kK5f2PHJfacKaUJlt5CGzTSfdO0Pp/aj0KXvBXPB5u0GbO3DilX+jZUqvKmSIPE+Gaek680OR1A//hOetbizPYpZudZyRpCUwi10oV/tMrGxP0sdAMCoPJ7dDT+yTbjFqZ46ul+jQYeqYGjdpjrv1xR+Gyerg8TIML5XKZjzVtjnJeJbM4yTdV4s/LrQG5gGAvtHtCNXUnf7SSj9fYnUI+y4CVy1lpMGoZnegapNQrLc08tRhnpVOYrm4WJ1hUzsFMyv9n0Z/7X241p5dfW2Y7SFR09VFAz4tU8S0wkVYpwaBMez2cW3DreMGORLeX5VcYTnd8aQnnq9ZZAWDcuJPjKrCHRusIiOiXzd3aCuqXDW2VdNLYSWBbMB9iL6DxnIRwgouWumJD9A/XdurNE1/OtYlsm5GmDLoVt3ExDsxgEx3bRoFlsmRFI+xlLXH7wQWoKjIYEfMiX7CUS8aDTMtKameQXXGo7UwFR18X9ez0CWrL8YBRAhikrTpt44lwyrLXKadrJcg8tZosQoMDoloQKls5gLbbYxX2AaiuMVjWqE0mVRDs++2+LZSvkI3o6Gnp6RcSNvd3R9mP2KJ+4q4QMZuIod7puk9AZucLdoyxc4U7uB2/vuIu3wfaBk9Pg2/b22Tnvo9dxQsLUatGXoHnvNx+JMUACboTU9JBkhUfKlKx4LvMeyn/wL6bRK2c4ybE+f/8lHUj3OTXrk9OfnJRpk3LMmtZW2fbHHTXH5+BFIk/ALOxQSvd/9EfeT/twzLhr/K/DG4+ad3JkaZ/AuH94aX1KDcsT2cUkz9fyvk7Vvi+Uc/BhGIDYBq0zNcdrWsvTy7VoF8rN+NjInTpiXVooIL5NTNZi/Dvl8GTmXtBpAc4yc9flf+w25G42+McIuGwtofFTzL82xZIXUsB/0ejhFt76ouzTmZKu17lWhM8U92uZqP84Dki+df2nW7LfgVCCBCO/T3NjkAicpJleV5WX4viNQ91Lsf7z/ZfEWqiwPJzegEnBqZDCEiqfJ13qVuWKLvpOTEiuzyIGJNDT5y0jcrolf2dRETWF3qIp4ppSEJ8+2c7FeSdkyZkpwUH3zGEnqOp2DN1FxPtr7hDnfstvz0NZ/tRrpMYvfGFX777+Ao02dP0YER8NkVPzmDEadv+pIToYx/vFzW1/ZyS9bGyh4I9rtX2dwMKtXQqxDGdRcRWL69ef1VbvD3Nat0cVh0IzhhT3KLg+njiUkTWYuEfIJZHE2bdyuAVqfZTVu2iVO37lxpllbSsMFbnfD+JpxBeAXEdH6ULk8g6fdBlS7Su/hMZkK6oyvHGB2FLiEzTaWrGUexDKHz+d8G8hFJtkdttjRd3x0F/++f2P06B06h2vtyEGmw010IdPgIV05/8CAAD//82LgnI=" + return "eJzUXE2T47bRvu+v6PLltd+a5V5Te0hVap2PrcSuza5TSeUiQ2BLhAUCNNCURv71KXyRFEVp9AFq1nPYnZFIPM8DNIBGd5NvYYP79yAsCf0GgARJfA/ffHR/f/MGoETLjWhIaPUe/vgGAMK18IMuW4lvAAxKZBbfwxKJvQFYCZSlfe8vfQuK1dg3735o3+B7WBvdNvGTCQz387O/62fgWhETyoIlRu4zboEqRrBDg2CQlbAyuoaPA5AhiSERLoiVKLvPp+icoeR+PkzQMSgZYQmkgSoMTOBDwAKLZis4DhoZd1n6GbMeMl+bhhc1UqXLg++Tgg3ud9qMvzujw/38VKFvGCYaPgA+1pALearlA2gHkhfX3Q565YfKoG2Qk9ieoDPJyyI3SAtnmEZLiWZht3zBOF9wg84QFhwNFVy3iiapS63W1/NWbb1E45i75sVKcEZoIWJC2aIzwKgAGPf44WuhVXFCitmiWRityZNe4HMjzH5hkWtV2kn2K6nZWNcF9FslnoFEjZZY3TyBUBBRnmBXoeomjCPjJcJOSAmeERbwbwSLBIKcSoVr5kdNKODM+vEUitAoJgGN0WZab2fNaIqKqVLi9HS6YYQ0MTkYo8+fPljgum4kutHRyttbgH4Cg2tmSonWumtty7n/1cCKCdkafJl9bdeFQY5iO6cEsGSQ1VCjtWyNbqkLkIeCLqNrMd98GFFdT3F1eLDcX8XTEjM0r1FEiFM9eJndCrUu3Kaj+L6obbFs+Qap+P9J3nr5C/Jxz4cPF/csSG7x1MoiRCLwbS2kFHFWf9eNi9+tK1bCElEBaxrpli6h1VuJW5QQJ+K1QzXVD7atLx26lTY1o/dQtsaTyaA/Cn4C29buj/C5QOuWqWHf3KQt935ymn5vsDWSEdweG2dHmkmJ+4c4VAEqkz/l/s3rVrgWB26Fbg1H/0foQxBDWSc52Ybl9rP+3i7RKHS+QocwJjrJZ4vGHk+MO9nERhOBibWpc9adb8UnpuadDPp2EwmhLDH1ktvnzyMFa6lCRXEFK2q0VaOlcLN8Fl9vy6QoA/a7Q+x3Q2zYKL1TbsLEacIIGDRaKHKLj/O7Lhf1WoLyiHHdUvyAtvqg1UqMueYX4fDe9Xj3kVdIO202bukv0ZJQvmdMKx8wGj32uzF2NlGotnq/EpLQPFTQEDebmDUj3LH9Q4UkzGwirCiRs8eORsLMJmIrDLVMRt/goVpG0PdJ8ivgvmBERixbwpopsUJL8ysKyO+OkbMICi7+A4wsykh4Wcgn9+Bh7DvALPQfs3lE6lfvFud9LtMqd3XRGM3RWm0K3KKihW2YKtLJ7dUOwImAO3js3NkWGa9AKK5rodbgqQKzUAlLem1YDYHmZefBi7U/9NB7iebuAJwuziw490n4Mk0XnIrPq7GKNbbStPANvpLN9ioCixSd2WIIz0S6WE4a7o1Kr7DQnBI6OwzX3Uj+925tUqzQf/f1rJduSJzoxDHnInle9+uslWO92RbI82Jns9xjPTeaqyVG6G1qMa+X5ZN+vV/VoAkf/evzP14gaBjhel9oteAVU+vp4Nzd9ESNNkS/I+D/WdDqg0eEitmwsnHnTk3naY8Iu7/M4tdWoOW4MG688uU1JsgziFg93SnQLvPo446LTbvERdzmQxZsQZqYnJXopo+IxjhtxF61Uu67fARTYTGeDo463zfMNJ843eC+aJuSUTYD/mvwnXsg2OGy0nozzP/CFOYURx+bKqRmuWxg8wfbE/Ktp9XhW4PfOaBLSeXtttO8xjhvT2QOEtMabfVmTGiOzIYDypTXOAplD7kTPt+QuG+MrpEqbO106wn6F72cRL05UD8AHrfdrXr4a5szPOLztanRYWKSqThWjdHP+3NcXtfN+hxI9Fv1jW7VhJaHuU6fU/+n2291lyZE5HSJJnj2e82kZY6ZWfEbFss94StayhfxG1560Dumne9kd4LPGOAMlxnGdszmEOIEsZBW//0N7jHvVxzdYzJfy/A22hCafFvd59giiBKV8+viSSAh9bn6YPHwkUBYXwxHGgaZOhCreJP73hfFslhNM9y8gKky+b1Td3ApUNHwjhO1g76JYqfNxnl7Re6SCmHHPZJKLCL5hAy7SvAKYj1mPEYFchcTz153Mc3+oPhiJOMs2cYIxUVz4mCUjWODaKDDSkTJsNVK8MRlYIC+cNPfdJjKd9+3Fs9rYk0zr5rYwaxpYMlsKLlzf0i2RHnLKMxQEXPMelQhc5LfJNHBcvAaM3O4Gp3v0ZNEX2Mm3kb7q5iTQ0J3T8xhY7PPzmGnn5+ik8PzsoLXmK3nLelFyjEIUFTaTrs6WXgPSabnBRxicrvEUqI7w7lTb5eXSO7HxRrmX3WmdDjU6zg+esGZYP2ChaSub4wmzfXc601EOe93/unTx/5KsXK/b0WJ5RNoqtDshO3uBG2cc6Zi0WW6a3qUOs+f6zJfrP1YZVf97HCOpPo7whokLDQG/bMEWsm9W6D+9tNPn1KeY1pFL7ewyFsjaB/LMWb2eNLTP4eLfoA+P6B1Sy2TC5I2bB7B/4/bhb+AbVxv1XWrUrteG/oDxegYcWJbIg2tCtUhHmR0V4rGC+X+jR2X2HOmlCZYegtt0Ej3TdP6eGo/Cl3wVjwfHNJmjN46pFzh21DBypsiDRLjm3kyYmlyOoD+sZ70DMaZ41OM2rOSNYSmEGqlC//IlY0B/VnoBgRA5c/oaPwTb8ZtTDEb6n6NBh6pgaN2mAO4XFH4bJ6uDxMgwvlYpmPNW2PcKhPZnGWaqvRm5deB3MAwFuA9oBu7Ur/bSUbr7TNTj7LgJXLWWkwahrk+g9QaheW5pxmjjJSV+cpmYaJ1xQzslMxvNn0twPV2Y1r5tXW2o3RFRw8V1Ix4Nc8WE0mVIiRtwmP7zILbxxtmLLKlPL/LeKLzW0MoW73eEgjrxrkEX5k1JFpXWESnZP7OTlC3dHirrPPGVgLLgnkHexEXz1kIB4houSsmZP+gfbfvLNH1fKvYlgl52qBLYRs309AsBs6xXTRoFltmBNJ+xtKXH7yTmhyjIQHv8iV7iUQ86LSMtGcm+QWX2s5UWNR1cf++j4CWLD8YBZBhyorTJp44lwxrrXKadrLcQ4vZIgQoMLoloYKlM1iLLXZ+H6DaCqNVjepEUCXRju8EsMWyFfIRPR0NPb064sbe7mj7MXvUmrirhAxm4ih3um6T0Bm5wt2jLFzhTu4Hb/W4i7fB9oGT0+Db9vbZOe8j2XFCwtRu0Zem+dVvPhJjgATcCKnpIcEKj5QpWPFc5k3Kf/AvrNErZzhpYX3+/ktKSPcxNeuD05+clGmTcsya1lbZzscdNcfn4AUjT8As7FBK939cj/w67d0x467xvw5vPGreyZGlfQLj/uGl9SE1LE9EF5M8X+P7OtX8vlDOwYdhAGIbtM7UHK9pLU8v16JdKDfj4yR36oh1aaGw+DYxWYv075TDk5l7QacFOMvMXa//sTuQu9ngHy/gsrWExk8x/zoVS15IAf9Fo4dHeOuLtU9HSrpe51oRPlM8r2Wi/uPYIfnW9Z9uyX4HQgkSjPw5zY1BInCSZnqNVV6K4zcRdS/L+s/3XxJrocL2cPoAJgWnQgpLqHycdKlblcu76DsxIbk+ixiQQE/nW0bkdEv+zqIiagq9RVPEPaUgPp3ZzsV5J2TJmSnBQffMYSeo6k4M3UXE+2vuEOd+y2/PQ1k+6zVS4ze+cKp3X3+BRhu6foyIj4bIqXnMGA27/9QQHYzj/eLmtr8zkl42tlDwx7Xavo5j4fYuhViGXFTcxeL+9We11fvDmNbtXsWh0IwuxT0KrvcnDmVkdSbuEXKJJ3H2bR1ug1ofRfUu2uWOX8VxZlvbCkNF7veGeBrxxSDX0VG6ELlWh0+6DKH2ld9CY7AVVRnexEBsKfEJGm2tWMo9COXTTyfWt+CKTTK77bGiLj30l39+/+M0KJ169+ttiMFmQw304ZNhIdz5vwAAAP//h0iIoA==" } diff --git a/x-pack/metricbeat/module/istio/istiod/_meta/docs.ascoodoc b/x-pack/metricbeat/module/istio/istiod/_meta/docs.ascoodoc new file mode 100644 index 00000000000..61170c60413 --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/_meta/docs.ascoodoc @@ -0,0 +1,4 @@ +This is the istiod metricset of the module istio. +This metricset collects metrics from the Istio Daemon of Istio versions higher than 1.5 + +Tested with Istio 1.7 diff --git a/x-pack/metricbeat/module/istio/istiod/_meta/fields.yml b/x-pack/metricbeat/module/istio/istiod/_meta/fields.yml new file mode 100644 index 00000000000..8033a27f5ac --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/_meta/fields.yml @@ -0,0 +1 @@ +- release: beta diff --git a/x-pack/metricbeat/module/istio/istiod/_meta/testdata/config.yml b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/config.yml new file mode 100644 index 00000000000..376691991b5 --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/config.yml @@ -0,0 +1,5 @@ +type: http +url: "/metrics" +suffix: plain +remove_fields_from_comparison: ["prometheus.labels.instance"] +omit_documented_fields_check: ["prometheus.labels.*"] diff --git a/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain new file mode 100644 index 00000000000..4ec26bf4a2c --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain @@ -0,0 +1,499 @@ +# HELP citadel_server_csr_count The number of CSRs received by Citadel server. +# TYPE citadel_server_csr_count counter +citadel_server_csr_count 8 +# HELP citadel_server_root_cert_expiry_timestamp The unix timestamp, in seconds, when Citadel root cert will expire. A negative time indicates the cert is expired. +# TYPE citadel_server_root_cert_expiry_timestamp gauge +citadel_server_root_cert_expiry_timestamp 1.916309303e+09 +# HELP citadel_server_success_cert_issuance_count The number of certificates issuances that have succeeded. +# TYPE citadel_server_success_cert_issuance_count counter +citadel_server_success_cert_issuance_count 8 +# HELP endpoint_no_pod Endpoints without an associated pod. +# TYPE endpoint_no_pod gauge +endpoint_no_pod 0 +# HELP galley_validation_config_updates k8s webhook configuration updates +# TYPE galley_validation_config_updates counter +galley_validation_config_updates 2 +# HELP galley_validation_failed Resource validation failed +# TYPE galley_validation_failed counter +galley_validation_failed{group="networking.istio.io",reason="invalid_resource",resource="gateways",version="v1alpha3"} 1 +# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 8.969e-06 +go_gc_duration_seconds{quantile="0.25"} 8.7539e-05 +go_gc_duration_seconds{quantile="0.5"} 0.000238037 +go_gc_duration_seconds{quantile="0.75"} 0.000783504 +go_gc_duration_seconds{quantile="1"} 0.005515511 +go_gc_duration_seconds_sum 0.013316174 +go_gc_duration_seconds_count 13 +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 345 +# HELP go_info Information about the Go environment. +# TYPE go_info gauge +go_info{version="go1.14.7"} 1 +# HELP go_memstats_alloc_bytes Number of bytes allocated and still in use. +# TYPE go_memstats_alloc_bytes gauge +go_memstats_alloc_bytes 3.1723136e+07 +# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed. +# TYPE go_memstats_alloc_bytes_total counter +go_memstats_alloc_bytes_total 1.106604e+08 +# HELP go_memstats_buck_hash_sys_bytes Number of bytes used by the profiling bucket hash table. +# TYPE go_memstats_buck_hash_sys_bytes gauge +go_memstats_buck_hash_sys_bytes 1.496287e+06 +# HELP go_memstats_frees_total Total number of frees. +# TYPE go_memstats_frees_total counter +go_memstats_frees_total 982627 +# HELP go_memstats_gc_cpu_fraction The fraction of this program's available CPU time used by the GC since the program started. +# TYPE go_memstats_gc_cpu_fraction gauge +go_memstats_gc_cpu_fraction 6.904294960464036e-05 +# HELP go_memstats_gc_sys_bytes Number of bytes used for garbage collection system metadata. +# TYPE go_memstats_gc_sys_bytes gauge +go_memstats_gc_sys_bytes 3.832072e+06 +# HELP go_memstats_heap_alloc_bytes Number of heap bytes allocated and still in use. +# TYPE go_memstats_heap_alloc_bytes gauge +go_memstats_heap_alloc_bytes 3.1723136e+07 +# HELP go_memstats_heap_idle_bytes Number of heap bytes waiting to be used. +# TYPE go_memstats_heap_idle_bytes gauge +go_memstats_heap_idle_bytes 2.1651456e+07 +# HELP go_memstats_heap_inuse_bytes Number of heap bytes that are in use. +# TYPE go_memstats_heap_inuse_bytes gauge +go_memstats_heap_inuse_bytes 4.2508288e+07 +# HELP go_memstats_heap_objects Number of allocated objects. +# TYPE go_memstats_heap_objects gauge +go_memstats_heap_objects 157553 +# HELP go_memstats_heap_released_bytes Number of heap bytes released to OS. +# TYPE go_memstats_heap_released_bytes gauge +go_memstats_heap_released_bytes 1.7825792e+07 +# HELP go_memstats_heap_sys_bytes Number of heap bytes obtained from system. +# TYPE go_memstats_heap_sys_bytes gauge +go_memstats_heap_sys_bytes 6.4159744e+07 +# HELP go_memstats_last_gc_time_seconds Number of seconds since 1970 of last garbage collection. +# TYPE go_memstats_last_gc_time_seconds gauge +go_memstats_last_gc_time_seconds 1.6019028053800597e+09 +# HELP go_memstats_lookups_total Total number of pointer lookups. +# TYPE go_memstats_lookups_total counter +go_memstats_lookups_total 0 +# HELP go_memstats_mallocs_total Total number of mallocs. +# TYPE go_memstats_mallocs_total counter +go_memstats_mallocs_total 1.14018e+06 +# HELP go_memstats_mcache_inuse_bytes Number of bytes in use by mcache structures. +# TYPE go_memstats_mcache_inuse_bytes gauge +go_memstats_mcache_inuse_bytes 6944 +# HELP go_memstats_mcache_sys_bytes Number of bytes used for mcache structures obtained from system. +# TYPE go_memstats_mcache_sys_bytes gauge +go_memstats_mcache_sys_bytes 16384 +# HELP go_memstats_mspan_inuse_bytes Number of bytes in use by mspan structures. +# TYPE go_memstats_mspan_inuse_bytes gauge +go_memstats_mspan_inuse_bytes 485384 +# HELP go_memstats_mspan_sys_bytes Number of bytes used for mspan structures obtained from system. +# TYPE go_memstats_mspan_sys_bytes gauge +go_memstats_mspan_sys_bytes 573440 +# HELP go_memstats_next_gc_bytes Number of heap bytes when next garbage collection will take place. +# TYPE go_memstats_next_gc_bytes gauge +go_memstats_next_gc_bytes 5.3766592e+07 +# HELP go_memstats_other_sys_bytes Number of bytes used for other system allocations. +# TYPE go_memstats_other_sys_bytes gauge +go_memstats_other_sys_bytes 914201 +# HELP go_memstats_stack_inuse_bytes Number of bytes in use by the stack allocator. +# TYPE go_memstats_stack_inuse_bytes gauge +go_memstats_stack_inuse_bytes 2.94912e+06 +# HELP go_memstats_stack_sys_bytes Number of bytes obtained from system for stack allocator. +# TYPE go_memstats_stack_sys_bytes gauge +go_memstats_stack_sys_bytes 2.94912e+06 +# HELP go_memstats_sys_bytes Number of bytes obtained from system. +# TYPE go_memstats_sys_bytes gauge +go_memstats_sys_bytes 7.3941248e+07 +# HELP go_threads Number of OS threads created. +# TYPE go_threads gauge +go_threads 14 +# HELP grpc_server_handled_total Total number of RPCs completed on the server, regardless of success or failure. +# TYPE grpc_server_handled_total counter +grpc_server_handled_total{grpc_code="Aborted",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Aborted",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Aborted",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Aborted",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Aborted",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Aborted",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="AlreadyExists",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Canceled",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DataLoss",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="DeadlineExceeded",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="FailedPrecondition",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Internal",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="InvalidArgument",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="NotFound",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OK",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 8 +grpc_server_handled_total{grpc_code="OK",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OK",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OK",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OK",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OK",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="OutOfRange",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="PermissionDenied",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="ResourceExhausted",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unauthenticated",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unavailable",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unimplemented",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handled_total{grpc_code="Unknown",grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +# HELP grpc_server_handling_seconds Histogram of response latency (seconds) of gRPC that had been application-level handled by the server. +# TYPE grpc_server_handling_seconds histogram +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.005"} 7 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.01"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.025"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.05"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.1"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.25"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="0.5"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="1"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="2.5"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="5"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="10"} 8 +grpc_server_handling_seconds_bucket{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary",le="+Inf"} 8 +grpc_server_handling_seconds_sum{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 0.031283576 +grpc_server_handling_seconds_count{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 8 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.005"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.025"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.05"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.25"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="2.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="10"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="+Inf"} 0 +grpc_server_handling_seconds_sum{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_count{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.005"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.025"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.05"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.25"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="2.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="10"} 0 +grpc_server_handling_seconds_bucket{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="+Inf"} 0 +grpc_server_handling_seconds_sum{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_count{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.005"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.025"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.05"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.25"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="0.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="2.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="10"} 0 +grpc_server_handling_seconds_bucket{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream",le="+Inf"} 0 +grpc_server_handling_seconds_sum{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_count{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.005"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.025"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.05"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.25"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="2.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="10"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream",le="+Inf"} 0 +grpc_server_handling_seconds_sum{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_count{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.005"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.01"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.025"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.05"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.25"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="0.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="1"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="2.5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="5"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="10"} 0 +grpc_server_handling_seconds_bucket{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream",le="+Inf"} 0 +grpc_server_handling_seconds_sum{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_handling_seconds_count{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +# HELP grpc_server_msg_received_total Total number of RPC stream messages received on the server. +# TYPE grpc_server_msg_received_total counter +grpc_server_msg_received_total{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 8 +grpc_server_msg_received_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_received_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_received_total{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_msg_received_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_received_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +# HELP grpc_server_msg_sent_total Total number of gRPC stream messages sent by the server. +# TYPE grpc_server_msg_sent_total counter +grpc_server_msg_sent_total{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 8 +grpc_server_msg_sent_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_sent_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_sent_total{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_msg_sent_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_msg_sent_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +# HELP grpc_server_started_total Total number of RPCs started on the server. +# TYPE grpc_server_started_total counter +grpc_server_started_total{grpc_method="CreateCertificate",grpc_service="istio.v1.auth.IstioCertificateService",grpc_type="unary"} 8 +grpc_server_started_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_started_total{grpc_method="DeltaAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_started_total{grpc_method="ServerReflectionInfo",grpc_service="grpc.reflection.v1alpha.ServerReflection",grpc_type="bidi_stream"} 0 +grpc_server_started_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v2.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +grpc_server_started_total{grpc_method="StreamAggregatedResources",grpc_service="envoy.service.discovery.v3.AggregatedDiscoveryService",grpc_type="bidi_stream"} 0 +# HELP istio_build Istio component build info +# TYPE istio_build gauge +istio_build{component="pilot",tag="1.7.1"} 1 +# HELP pilot_conflict_inbound_listener Number of conflicting inbound listeners. +# TYPE pilot_conflict_inbound_listener gauge +pilot_conflict_inbound_listener 0 +# HELP pilot_conflict_outbound_listener_http_over_current_tcp Number of conflicting wildcard http listeners with current wildcard tcp listener. +# TYPE pilot_conflict_outbound_listener_http_over_current_tcp gauge +pilot_conflict_outbound_listener_http_over_current_tcp 0 +# HELP pilot_conflict_outbound_listener_tcp_over_current_http Number of conflicting wildcard tcp listeners with current wildcard http listener. +# TYPE pilot_conflict_outbound_listener_tcp_over_current_http gauge +pilot_conflict_outbound_listener_tcp_over_current_http 0 +# HELP pilot_conflict_outbound_listener_tcp_over_current_tcp Number of conflicting tcp listeners with current tcp listener. +# TYPE pilot_conflict_outbound_listener_tcp_over_current_tcp gauge +pilot_conflict_outbound_listener_tcp_over_current_tcp 0 +# HELP pilot_destrule_subsets Duplicate subsets across destination rules for same host +# TYPE pilot_destrule_subsets gauge +pilot_destrule_subsets 0 +# HELP pilot_duplicate_envoy_clusters Duplicate envoy clusters caused by service entries with same hostname +# TYPE pilot_duplicate_envoy_clusters gauge +pilot_duplicate_envoy_clusters 0 +# HELP pilot_eds_no_instances Number of clusters without instances. +# TYPE pilot_eds_no_instances gauge +pilot_eds_no_instances 0 +# HELP pilot_endpoint_not_ready Endpoint found in unready state. +# TYPE pilot_endpoint_not_ready gauge +pilot_endpoint_not_ready 0 +# HELP pilot_inbound_updates Total number of updates received by pilot. +# TYPE pilot_inbound_updates counter +pilot_inbound_updates{type="config"} 90 +pilot_inbound_updates{type="eds"} 54 +pilot_inbound_updates{type="svc"} 22 +# HELP pilot_info Pilot version and build information. +# TYPE pilot_info gauge +pilot_info{version="1.7.1-4e26c697ce460dc8d3b25b25818fb0163ca16394-Clean"} 1 +# HELP pilot_k8s_cfg_events Events from k8s config. +# TYPE pilot_k8s_cfg_events counter +pilot_k8s_cfg_events{event="add",type="DestinationRule"} 4 +pilot_k8s_cfg_events{event="add",type="EnvoyFilter"} 8 +pilot_k8s_cfg_events{event="add",type="Gateway"} 1 +pilot_k8s_cfg_events{event="add",type="VirtualService"} 1 +# HELP pilot_k8s_reg_events Events from k8s registry. +# TYPE pilot_k8s_reg_events counter +pilot_k8s_reg_events{event="add",type="Endpoints"} 12 +pilot_k8s_reg_events{event="add",type="Nodes"} 1 +pilot_k8s_reg_events{event="add",type="Pods"} 22 +pilot_k8s_reg_events{event="add",type="Services"} 11 +pilot_k8s_reg_events{event="update",type="Endpoints"} 30 +pilot_k8s_reg_events{event="update",type="Nodes"} 1 +pilot_k8s_reg_events{event="update",type="Pods"} 61 +pilot_k8s_reg_events{event="updatesame",type="Endpoints"} 185 +# HELP pilot_no_ip Pods not found in the endpoint table, possibly invalid. +# TYPE pilot_no_ip gauge +pilot_no_ip 0 +# HELP pilot_proxy_convergence_time Delay in seconds between config change and a proxy receiving all required configuration. +# TYPE pilot_proxy_convergence_time histogram +pilot_proxy_convergence_time_bucket{le="0.1"} 1 +pilot_proxy_convergence_time_bucket{le="0.5"} 1 +pilot_proxy_convergence_time_bucket{le="1"} 1 +pilot_proxy_convergence_time_bucket{le="3"} 1 +pilot_proxy_convergence_time_bucket{le="5"} 1 +pilot_proxy_convergence_time_bucket{le="10"} 1 +pilot_proxy_convergence_time_bucket{le="20"} 1 +pilot_proxy_convergence_time_bucket{le="30"} 1 +pilot_proxy_convergence_time_bucket{le="+Inf"} 1 +pilot_proxy_convergence_time_sum 0.002034992 +pilot_proxy_convergence_time_count 1 +# HELP pilot_proxy_queue_time Time in seconds, a proxy is in the push queue before being dequeued. +# TYPE pilot_proxy_queue_time histogram +pilot_proxy_queue_time_bucket{le="0.1"} 41 +pilot_proxy_queue_time_bucket{le="1"} 41 +pilot_proxy_queue_time_bucket{le="3"} 41 +pilot_proxy_queue_time_bucket{le="5"} 41 +pilot_proxy_queue_time_bucket{le="10"} 41 +pilot_proxy_queue_time_bucket{le="20"} 41 +pilot_proxy_queue_time_bucket{le="30"} 41 +pilot_proxy_queue_time_bucket{le="+Inf"} 41 +pilot_proxy_queue_time_sum 0.002216352 +pilot_proxy_queue_time_count 41 +# HELP pilot_push_triggers Total number of times a push was triggered, labeled by reason for the push. +# TYPE pilot_push_triggers counter +pilot_push_triggers{type="endpoint"} 40 +pilot_push_triggers{type="proxy"} 1 +# HELP pilot_services Total services known to pilot. +# TYPE pilot_services gauge +pilot_services 11 +# HELP pilot_virt_services Total virtual services known to pilot. +# TYPE pilot_virt_services gauge +pilot_virt_services 1 +# HELP pilot_vservice_dup_domain Virtual services with dup domains. +# TYPE pilot_vservice_dup_domain gauge +pilot_vservice_dup_domain 0 +# HELP pilot_xds Number of endpoints connected to this pilot using XDS. +# TYPE pilot_xds gauge +pilot_xds{version="1.7.1"} 8 +# HELP pilot_xds_push_time Total time in seconds Pilot takes to push lds, rds, cds and eds. +# TYPE pilot_xds_push_time histogram +pilot_xds_push_time_bucket{type="cds",le="0.01"} 9 +pilot_xds_push_time_bucket{type="cds",le="0.1"} 9 +pilot_xds_push_time_bucket{type="cds",le="1"} 9 +pilot_xds_push_time_bucket{type="cds",le="3"} 9 +pilot_xds_push_time_bucket{type="cds",le="5"} 9 +pilot_xds_push_time_bucket{type="cds",le="10"} 9 +pilot_xds_push_time_bucket{type="cds",le="20"} 9 +pilot_xds_push_time_bucket{type="cds",le="30"} 9 +pilot_xds_push_time_bucket{type="cds",le="+Inf"} 9 +pilot_xds_push_time_sum{type="cds"} 0.009549363999999998 +pilot_xds_push_time_count{type="cds"} 9 +pilot_xds_push_time_bucket{type="eds",le="0.01"} 49 +pilot_xds_push_time_bucket{type="eds",le="0.1"} 49 +pilot_xds_push_time_bucket{type="eds",le="1"} 49 +pilot_xds_push_time_bucket{type="eds",le="3"} 49 +pilot_xds_push_time_bucket{type="eds",le="5"} 49 +pilot_xds_push_time_bucket{type="eds",le="10"} 49 +pilot_xds_push_time_bucket{type="eds",le="20"} 49 +pilot_xds_push_time_bucket{type="eds",le="30"} 49 +pilot_xds_push_time_bucket{type="eds",le="+Inf"} 49 +pilot_xds_push_time_sum{type="eds"} 0.008623654 +pilot_xds_push_time_count{type="eds"} 49 +pilot_xds_push_time_bucket{type="lds",le="0.01"} 9 +pilot_xds_push_time_bucket{type="lds",le="0.1"} 9 +pilot_xds_push_time_bucket{type="lds",le="1"} 9 +pilot_xds_push_time_bucket{type="lds",le="3"} 9 +pilot_xds_push_time_bucket{type="lds",le="5"} 9 +pilot_xds_push_time_bucket{type="lds",le="10"} 9 +pilot_xds_push_time_bucket{type="lds",le="20"} 9 +pilot_xds_push_time_bucket{type="lds",le="30"} 9 +pilot_xds_push_time_bucket{type="lds",le="+Inf"} 9 +pilot_xds_push_time_sum{type="lds"} 0.011886486000000002 +pilot_xds_push_time_count{type="lds"} 9 +pilot_xds_push_time_bucket{type="rds",le="0.01"} 9 +pilot_xds_push_time_bucket{type="rds",le="0.1"} 9 +pilot_xds_push_time_bucket{type="rds",le="1"} 9 +pilot_xds_push_time_bucket{type="rds",le="3"} 9 +pilot_xds_push_time_bucket{type="rds",le="5"} 9 +pilot_xds_push_time_bucket{type="rds",le="10"} 9 +pilot_xds_push_time_bucket{type="rds",le="20"} 9 +pilot_xds_push_time_bucket{type="rds",le="30"} 9 +pilot_xds_push_time_bucket{type="rds",le="+Inf"} 9 +pilot_xds_push_time_sum{type="rds"} 0.002646734 +pilot_xds_push_time_count{type="rds"} 9 +# HELP pilot_xds_pushes Pilot build and send errors for lds, rds, cds and eds. +# TYPE pilot_xds_pushes counter +pilot_xds_pushes{type="cds"} 9 +pilot_xds_pushes{type="eds"} 49 +pilot_xds_pushes{type="lds"} 9 +pilot_xds_pushes{type="rds"} 9 +# HELP process_cpu_seconds_total Total user and system CPU time spent in seconds. +# TYPE process_cpu_seconds_total counter +process_cpu_seconds_total 2.35 +# HELP process_max_fds Maximum number of open file descriptors. +# TYPE process_max_fds gauge +process_max_fds 1.048576e+06 +# HELP process_open_fds Number of open file descriptors. +# TYPE process_open_fds gauge +process_open_fds 52 +# HELP process_resident_memory_bytes Resident memory size in bytes. +# TYPE process_resident_memory_bytes gauge +process_resident_memory_bytes 1.08478464e+08 +# HELP process_start_time_seconds Start time of the process since unix epoch in seconds. +# TYPE process_start_time_seconds gauge +process_start_time_seconds 1.60190238401e+09 +# HELP process_virtual_memory_bytes Virtual memory size in bytes. +# TYPE process_virtual_memory_bytes gauge +process_virtual_memory_bytes 8.08730624e+08 +# HELP process_virtual_memory_max_bytes Maximum amount of virtual memory available in bytes. +# TYPE process_virtual_memory_max_bytes gauge +process_virtual_memory_max_bytes -1 +# HELP statsd_metric_mapper_cache_gets_total The count of total metric cache gets. +# TYPE statsd_metric_mapper_cache_gets_total counter +statsd_metric_mapper_cache_gets_total 0 +# HELP statsd_metric_mapper_cache_hits_total The count of total metric cache hits. +# TYPE statsd_metric_mapper_cache_hits_total counter +statsd_metric_mapper_cache_hits_total 0 +# HELP statsd_metric_mapper_cache_length The count of unique metrics currently cached. +# TYPE statsd_metric_mapper_cache_length gauge +statsd_metric_mapper_cache_length 0 diff --git a/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain-expected.json b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain-expected.json new file mode 100644 index 00000000000..2d96ee6b90e --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/_meta/testdata/istiod.v1.7.1.plain-expected.json @@ -0,0 +1,843 @@ +[ + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Gateway" + }, + "pilot_k8s_cfg_events": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Nodes" + }, + "pilot_k8s_reg_events": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "update", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Nodes" + }, + "pilot_k8s_reg_events": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "DestinationRule" + }, + "pilot_k8s_cfg_events": { + "counter": 4, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "proxy" + }, + "pilot_push_triggers": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "lds" + }, + "pilot_xds_push_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.005, + 0.05500000000000001, + 0.55, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_xds_pushes": { + "counter": 9, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "svc" + }, + "pilot_inbound_updates": { + "counter": 22, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "eds" + }, + "pilot_inbound_updates": { + "counter": 54, + "rate": 0 + }, + "pilot_xds_push_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.005, + 0.05500000000000001, + 0.55, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_xds_pushes": { + "counter": 49, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "EnvoyFilter" + }, + "pilot_k8s_cfg_events": { + "counter": 8, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "endpoint" + }, + "pilot_push_triggers": { + "counter": 40, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "version": "1.7.1" + }, + "pilot_xds": { + "value": 8 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Services" + }, + "pilot_k8s_reg_events": { + "counter": 11, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "update", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Pods" + }, + "pilot_k8s_reg_events": { + "counter": 61, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "update", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Endpoints" + }, + "pilot_k8s_reg_events": { + "counter": 30, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "version": "1.7.1-4e26c697ce460dc8d3b25b25818fb0163ca16394-Clean" + }, + "pilot_info": { + "value": 1 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Endpoints" + }, + "pilot_k8s_reg_events": { + "counter": 12, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "VirtualService" + }, + "pilot_k8s_cfg_events": { + "counter": 1, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "cds" + }, + "pilot_xds_push_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.005, + 0.05500000000000001, + 0.55, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_xds_pushes": { + "counter": 9, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "add", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Pods" + }, + "pilot_k8s_reg_events": { + "counter": 22, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "event": "updatesame", + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "Endpoints" + }, + "pilot_k8s_reg_events": { + "counter": 185, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "config" + }, + "pilot_inbound_updates": { + "counter": 90, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "citadel_server_csr_count": { + "counter": 8, + "rate": 0 + }, + "citadel_server_root_cert_expiry_timestamp": { + "value": 1916309303 + }, + "citadel_server_success_cert_issuance_count": { + "counter": 8, + "rate": 0 + }, + "galley_validation_config_updates": { + "counter": 2, + "rate": 0 + }, + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio" + }, + "pilot_conflict_inbound_listener": { + "value": 0 + }, + "pilot_conflict_outbound_listener_http_over_current_tcp": { + "value": 0 + }, + "pilot_conflict_outbound_listener_tcp_over_current_http": { + "value": 0 + }, + "pilot_conflict_outbound_listener_tcp_over_current_tcp": { + "value": 0 + }, + "pilot_destrule_subsets": { + "value": 0 + }, + "pilot_duplicate_envoy_clusters": { + "value": 0 + }, + "pilot_eds_no_instances": { + "value": 0 + }, + "pilot_endpoint_not_ready": { + "value": 0 + }, + "pilot_no_ip": { + "value": 0 + }, + "pilot_proxy_convergence_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.05, + 0.30000000000000004, + 0.75, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_proxy_queue_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.05, + 0.55, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_services": { + "value": 11 + }, + "pilot_virt_services": { + "value": 1 + }, + "pilot_vservice_dup_domain": { + "value": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "labels": { + "instance": "127.0.0.1:51143", + "job": "istio", + "type": "rds" + }, + "pilot_xds_push_time": { + "histogram": { + "counts": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "values": [ + 0.005, + 0.05500000000000001, + 0.55, + 2, + 4, + 7.5, + 15, + 25, + 40 + ] + } + }, + "pilot_xds_pushes": { + "counter": 9, + "rate": 0 + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + }, + { + "event": { + "dataset": "istio.istiod", + "duration": 115000, + "module": "istio" + }, + "metricset": { + "name": "istiod", + "period": 10000 + }, + "prometheus": { + "galley_validation_failed": { + "counter": 1, + "rate": 0 + }, + "labels": { + "group": "networking.istio.io", + "instance": "127.0.0.1:51143", + "job": "istio", + "reason": "invalid_resource", + "resource": "gateways", + "version": "v1alpha3" + } + }, + "service": { + "address": "127.0.0.1:55555", + "type": "istio" + } + } +] \ No newline at end of file diff --git a/x-pack/metricbeat/module/istio/istiod/istiod_test.go b/x-pack/metricbeat/module/istio/istiod/istiod_test.go new file mode 100644 index 00000000000..23ce86ca7f0 --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/istiod_test.go @@ -0,0 +1,32 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// +build !integration + +package istiod + +import ( + "os" + "testing" + + "github.com/elastic/beats/v7/libbeat/logp" + "github.com/elastic/beats/v7/metricbeat/mb" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + + // Register input module and metricset + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/prometheus/collector" +) + +func init() { + // To be moved to some kind of helper + os.Setenv("BEAT_STRICT_PERMS", "false") + mb.Registry.SetSecondarySource(mb.NewLightModulesSource("../../../module")) +} + +func TestEventMapping(t *testing.T) { + logp.TestingSetup() + + mbtest.TestDataFiles(t, "istio", "istiod") +} diff --git a/x-pack/metricbeat/module/istio/istiod/manifest.yml b/x-pack/metricbeat/module/istio/istiod/manifest.yml new file mode 100644 index 00000000000..0a8294d9192 --- /dev/null +++ b/x-pack/metricbeat/module/istio/istiod/manifest.yml @@ -0,0 +1,11 @@ +default: false +input: + module: prometheus + metricset: collector + defaults: + metrics_path: /metrics + metrics_filters: + include: ["citadel_*", "galley_*", "pilot_*"] + exclude: ["^up$"] + use_types: true + rate_counters: true diff --git a/x-pack/metricbeat/module/istio/module.yaml b/x-pack/metricbeat/module/istio/module.yaml deleted file mode 100644 index 8b137891791..00000000000 --- a/x-pack/metricbeat/module/istio/module.yaml +++ /dev/null @@ -1 +0,0 @@ - diff --git a/x-pack/metricbeat/module/istio/module.yml b/x-pack/metricbeat/module/istio/module.yml new file mode 100644 index 00000000000..de29503fca5 --- /dev/null +++ b/x-pack/metricbeat/module/istio/module.yml @@ -0,0 +1,3 @@ +name: istio +metricsets: +- istiod diff --git a/x-pack/metricbeat/modules.d/aws.yml.disabled b/x-pack/metricbeat/modules.d/aws.yml.disabled index d0053297885..cc3103643c7 100644 --- a/x-pack/metricbeat/modules.d/aws.yml.disabled +++ b/x-pack/metricbeat/modules.d/aws.yml.disabled @@ -47,4 +47,8 @@ period: 24h metricsets: - s3_daily_storage +- module: aws + period: 1m + latency: 5m + metricsets: - s3_request diff --git a/x-pack/metricbeat/modules.d/azure.yml.disabled b/x-pack/metricbeat/modules.d/azure.yml.disabled index 23211f47206..10d00e003cf 100644 --- a/x-pack/metricbeat/modules.d/azure.yml.disabled +++ b/x-pack/metricbeat/modules.d/azure.yml.disabled @@ -114,3 +114,11 @@ # api_key: '' # metrics: # - id: ["requests/count", "requests/duration"] + +#- module: azure +# metricsets: +# - app_state +# enabled: true +# period: 300s +# application_id: '' +# api_key: '' diff --git a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled index ae540f20cfc..22a600e51d8 100644 --- a/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled +++ b/x-pack/metricbeat/modules.d/cloudfoundry.yml.disabled @@ -10,3 +10,4 @@ api_address: '${CLOUDFOUNDRY_API_ADDRESS:""}' client_id: '${CLOUDFOUNDRY_CLIENT_ID:""}' client_secret: '${CLOUDFOUNDRY_CLIENT_SECRET:""}' + shard_id: metricbeat diff --git a/x-pack/metricbeat/modules.d/istio.yml.disabled b/x-pack/metricbeat/modules.d/istio.yml.disabled index 1140d047a09..b65bcdef949 100644 --- a/x-pack/metricbeat/modules.d/istio.yml.disabled +++ b/x-pack/metricbeat/modules.d/istio.yml.disabled @@ -35,3 +35,10 @@ period: 10s # use istio-pilot.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset hosts: ["localhost:15014"] + +# Istio istiod to monitor the Istio Daemon for versions after 1.5 of Istio. +- module: istio + metricsets: ['istiod'] + period: 10s + # use istiod.istio-system:15014, when deploying Metricbeat in a kubernetes cluster as Pod or Daemonset + hosts: ['localhost:15014'] diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index 41257081e1f..fd30546f70a 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -18,3 +18,14 @@ stages: withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/winlogbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags diff --git a/x-pack/winlogbeat/Jenkinsfile.yml b/x-pack/winlogbeat/Jenkinsfile.yml index 396d1f03a7c..788d4a35369 100644 --- a/x-pack/winlogbeat/Jenkinsfile.yml +++ b/x-pack/winlogbeat/Jenkinsfile.yml @@ -18,3 +18,14 @@ stages: withModule: true platforms: ## override default labels in this specific stage. - "windows-2019" + windows-2016: + mage: "mage build unitTest" + platforms: ## override default labels in this specific stage. + - "windows-2016" + when: ## Override the top-level when. + comments: + - "/test x-pack/winlogbeat for windows-2016" + labels: + - "windows-2016" + branches: true ## for all the branches + tags: true ## for all the tags