Merge branch 'master' into new-js-client

Author: delvedor

.ci/.jenkins_edge_nodejs.yml

@@ -0,0 +1,6 @@
+NODEJS_VERSION:
+  - "12"
+  - "11"
+  - "10"
+  - "8"
+  - "6"

.ci/docker/docker-compose-edge.yml

@@ -0,0 +1,64 @@
+version: '2.1'
+
+services:
+  cassandra:
+    extends:
+      file: docker-compose.yml
+      service: cassandra
+  elasticsearch:
+    extends:
+      file: docker-compose.yml
+      service: elasticsearch
+  mongodb:
+    extends:
+      file: docker-compose.yml
+      service: mongodb
+  mssql:
+    extends:
+      file: docker-compose.yml
+      service: mssql
+  mysql:
+    extends:
+      file: docker-compose.yml
+      service: mysql
+  postgres:
+    extends:
+      file: docker-compose.yml
+      service: postgres
+  redis:
+    extends:
+      file: docker-compose.yml
+      service: redis
+  node_tests:
+    extends:
+      file: docker-compose-node-edge-test.yml
+      service: node_tests
+    depends_on:
+      cassandra:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+      mongodb:
+        condition: service_healthy
+      mssql:
+        condition: service_healthy
+      mysql:
+        condition: service_healthy
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+
+volumes:
+  nodepgdata:
+    driver: local
+  nodemongodata:
+    driver: local
+  nodemssqldata:
+    driver: local
+  nodemysqldata:
+    driver: local
+  nodeesdata:
+    driver: local
+  nodecassandradata:
+    driver: local

.ci/docker/docker-compose-node-edge-test.yml

@@ -0,0 +1,31 @@
+version: '2.1'
+
+services:
+  node_tests:
+    build:
+      context: node-edge-container
+      dockerfile: Dockerfile
+      args:
+        - NODE_VERSION=${NODE_VERSION}
+        - NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR}
+        - ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}
+    working_dir: /app
+    command: >
+      /bin/bash -c ".ci/scripts/docker-test.sh"
+    environment:
+      MONGODB_HOST: 'mongodb'
+      REDIS_HOST: 'redis'
+      ES_HOST: 'elasticsearch'
+      MSSQL_HOST: 'mssql'
+      MYSQL_HOST: 'mysql'
+      CASSANDRA_HOST: 'cassandra'
+      PGHOST: 'postgres'
+      PGUSER: 'postgres'
+      NODE_VERSION: ${NODE_VERSION}
+      NVM_NODEJS_ORG_MIRROR: ${NVM_NODEJS_ORG_MIRROR}
+      ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS}
+      TAV: ${TAV_VERSIONS}
+      HOME: /tmp
+    volumes:
+      - ${PWD}:/app
+    user: ${USER_ID}

.ci/docker/docker-compose-node-test.yml

@@ -21,6 +21,7 @@ services:
       PGUSER: 'postgres'
       NODE_VERSION: ${NODE_VERSION}
       TAV: ${TAV_VERSIONS}
+      ELASTIC_APM_ASYNC_HOOKS: ${ELASTIC_APM_ASYNC_HOOKS}
       HOME: /tmp
       PATH: /app/node_modules/.bin:./node_modules/.bin:/app/node_modules:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin
     volumes:

.ci/docker/node-container/Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /npm
 RUN ( [ "${NODE_VERSION%%.*}" -le 8  ] \
   && echo "Node.js ${NODE_VERSION} - Manual install npm" \
   && mkdir -p /npm/node_modules \
-  && npm install npm \
+  && npm install npm@6.9.0 \
   && rm /usr/local/bin/npm \
   && ln -s /npm/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm ) || exit 0
 

.ci/docker/node-edge-container/Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:18.04
+
+# Install nvm dependencies
+RUN apt-get -qq update \
+  && apt-get -qq install -y build-essential \
+  && apt-get -qq install -y libssl-dev \
+  && apt-get -qq install -y curl \
+  && apt-get -qq clean \
+  && rm -fr /var/lib/apt/lists/*
+
+# Environment variables to set what version to be installed
+ARG NODE_VERSION
+ARG NVM_NODEJS_ORG_MIRROR
+ARG ELASTIC_APM_ASYNC_HOOKS
+ENV NODE_VERSION=${NODE_VERSION}
+ENV NVM_NODEJS_ORG_MIRROR=${NVM_NODEJS_ORG_MIRROR}
+ENV ELASTIC_APM_ASYNC_HOOKS=${ELASTIC_APM_ASYNC_HOOKS}
+
+# nvm environment variables
+ENV NVM_VERSION v0.34.0
+ENV NVM_DIR /usr/local/nvm
+RUN mkdir $NVM_DIR
+RUN curl --silent -o- https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh | bash
+
+ENV NODE_PATH $NVM_DIR/versions/node/v$NODE_VERSION/lib/node_modules
+ENV PATH $NVM_DIR/versions/node/v$NODE_VERSION/bin:$PATH
+
+# Install node
+RUN echo "source $NVM_DIR/nvm.sh && \
+    nvm install $NODE_VERSION && \
+    nvm alias default $NODE_VERSION && \
+    nvm use default" | bash
+
+# Create symlinks
+RUN echo "source $NVM_DIR/nvm.sh && \
+    nvm version > /tmp/version" | bash
+RUN ln -fs $NVM_DIR/$(cat /tmp/version) $NVM_DIR/v$NODE_VERSION && \
+    ln -fs $NVM_DIR/versions/node/$(cat /tmp/version) $NVM_DIR/versions/node/v$NODE_VERSION
+
+RUN ( [ "${NODE_VERSION%%.*}" -le 8  ] \
+  && echo "Node.js ${NODE_VERSION} - Manual install npm" \
+  && mkdir -p /npm/node_modules \
+  && npm install npm@6.9.0 \
+  && rm /usr/local/bin/npm \
+  && ln -s /npm/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm ) || exit 0
+
+RUN ( [ "${NODE_VERSION%%.*}" -gt 8  ] \
+  && echo "Node.js ${NODE_VERSION} - upgrade npm" \
+  && npm install -g npm@latest) || exit 0
+
+# test npm works properly
+RUN node --version && npm --version
+
+# Set the PATH again
+ENV PATH /app/node_modules/.bin:./node_modules/.bin:/app/node_modules:$PATH
+
+WORKDIR /app

.ci/downstreamTests.groovy

@@ -13,7 +13,8 @@ it is need as field to store the results of the tests.
 pipeline {
   agent any
   environment {
-    BASE_DIR="src/github.com/elastic/apm-agent-nodejs"
+    REPO = 'apm-agent-nodejs'
+    BASE_DIR="src/github.com/elastic/${REPO}"
     PIPELINE_LOG_LEVEL='INFO'
     NOTIFY_TO = credentials('notify-to')
     JOB_GCS_BUCKET = credentials('gcs-bucket')
@@ -30,9 +31,9 @@ pipeline {
     quietPeriod(10)
   }
   parameters {
-    string(name: 'NODE_VERSION', defaultValue: "11", description: "Node.js version to test")
+    string(name: 'NODE_VERSION', defaultValue: "12", description: "Node.js version to test")
     string(name: 'BRANCH_SPECIFIER', defaultValue: "master", description: "Git branch/tag to use")
-    string(name: 'CHANGE_TARGET', defaultValue: "master", description: "Git branch/tag to merge before building")
+    string(name: 'MERGE_TARGET', defaultValue: "master", description: "Git branch/tag to merge before building")
   }
   stages {
     /**
@@ -47,7 +48,7 @@ pipeline {
           branch: "${params.BRANCH_SPECIFIER}",
           repo: "${REPO}",
           credentialsId: "${JOB_GIT_CREDENTIALS}",
-          mergeTarget: "${params.CHANGE_TARGET}"
+          mergeTarget: "${params.MERGE_TARGET}",
           reference: '/var/lib/jenkins/apm-agent-nodejs.git')
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
       }
@@ -82,29 +83,22 @@ pipeline {
     }
   }
   post {
-    always{
+    cleanup{
       script{
         if(nodeTasksGen?.results){
           writeJSON(file: 'results.json', json: toJSON(nodeTasksGen.results), pretty: 2)
           def mapResults = ["${params.agent_integration_test}": nodeTasksGen.results]
           def processor = new ResultsProcessor()
           processor.processResults(mapResults)
           archiveArtifacts allowEmptyArchive: true, artifacts: 'results.json,results.html', defaultExcludes: false
+          catchError(buildResult: 'SUCCESS') {
+            def datafile = readFile(file: "results.json")
+            def json = getVaultSecret(secret: 'secret/apm-team/ci/jenkins-stats-cloud')
+            sendDataToElasticsearch(es: json.data.url, data: datafile, restCall: '/jenkins-builds-nodejs-test-results/_doc/')
+          }
         }
       }
-    }
-    success {
-      echoColor(text: '[SUCCESS]', colorfg: 'green', colorbg: 'default')
-    }
-    aborted {
-      echoColor(text: '[ABORTED]', colorfg: 'magenta', colorbg: 'default')
-    }
-    failure {
-      echoColor(text: '[FAILURE]', colorfg: 'red', colorbg: 'default')
-      step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${NOTIFY_TO}", sendToIndividuals: false])
-    }
-    unstable {
-      echoColor(text: '[UNSTABLE]', colorfg: 'yellow', colorbg: 'default')
+      notifyBuildResult()
     }
   }
 }
@@ -130,9 +124,9 @@ class NodeParallelTaskGenerator extends DefaultParallelTaskGenerator {
           saveResult(x, y, 1)
         } catch(e){
           saveResult(x, y, 0)
-          error("${label} tests failed : ${e.toString()}\n")
+          steps.error("${label} tests failed : ${e.toString()}\n")
         } finally {
-          wrappingUp()
+          steps.wrappingUp()
         }
       }
     }
@@ -158,9 +152,9 @@ def runScript(Map params = [:]){
   Collect test results and report to Codecov
 */
 def wrappingUp(){
-  docker.image('node:11').inside("-v ${WORKSPACE}/${BASE_DIR}:/app"){
-    steps.sh(label: "Convert Test results to JUnit format", script: 'cd /app && .ci/convert_tap_to_junit.sh')
+  docker.image('node:12').inside("-v ${WORKSPACE}/${BASE_DIR}:/app"){
+    sh(label: "Convert Test results to JUnit format", script: 'cd /app && .ci/scripts/convert_tap_to_junit.sh')
   }
   junit(allowEmptyResults: true, keepLongStdio: true, testResults: "${BASE_DIR}/**/junit-*.xml")
-  codecov(repo: 'apm-agent-nodejs', basedir: "${BASE_DIR}", secret: "${CODECOV_SECRET}")
+  codecov(repo: env.REPO, basedir: "${BASE_DIR}", secret: "${CODECOV_SECRET}")
 }

.ci/jobs/apm-agent-nodejs-downstream.yml

@@ -0,0 +1,36 @@
+---
+- job:
+    name: apm-agent-nodejs/apm-agent-nodejs-downstream
+    display-name: APM Agent Node.js Downstream
+    description: APM Agent Node.js Downstream
+    script-path: .ci/downstreamTests.groovy
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: true
+        property-strategies:
+          all-branches:
+          - suppress-scm-triggering: true
+        repo: apm-agent-nodejs
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'

.ci/jobs/apm-agent-nodejs-mbp.yml

@@ -3,3 +3,39 @@
     name: apm-agent-nodejs/apm-agent-nodejs-mbp
     display-name: APM Agent Node.js
     description: APM Agent Node.js
+    script-path: Jenkinsfile
+    scm:
+    - github:
+        branch-discovery: no-pr
+        discover-pr-forks-strategy: merge-current
+        discover-pr-forks-trust: permission
+        discover-pr-origin: merge-current
+        discover-tags: true
+        head-filter-regex: '^(?!greenkeeper).*$'
+        repo: apm-agent-nodejs
+        repo-owner: elastic
+        credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
+        ssh-checkout:
+          credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+        build-strategies:
+        - tags:
+            ignore-tags-older-than: -1
+            ignore-tags-newer-than: -1
+        - regular-branches: true
+        - change-request:
+            ignore-target-only-changes: false
+        clean:
+          after: true
+          before: true
+        prune: true
+        shallow-clone: true
+        depth: 3
+        do-not-fetch-tags: true
+        submodule:
+          disable: false
+          recursive: true
+          parent-credentials: true
+          timeout: 100
+        timeout: '15'
+        use-author: true
+        wipe-workspace: 'True'

.ci/jobs/apm-agent-nodejs-schedule-daily.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: apm-agent-nodejs/apm-agent-nodejs-schedule-daily
+    display-name: Node.js Jobs scheduled daily
+    description: Node.js Jobs scheduled daily from Monday to Friday
+    view: APM-CI
+    project-type: pipeline
+    parameters:
+    - string:
+        name: branch_specifier
+        default: master
+        description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-daily.groovy
+      scm:
+      - git:
+          url: git@github.com:elastic/apm-agent-nodejs.git
+          refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+          wipe-workspace: 'True'
+          name: origin
+          shallow-clone: true
+          credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+          reference-repo: /var/lib/jenkins/.git-references/apm-agent-nodejs.git
+          branches:
+          - $branch_specifier
+    triggers:
+    - timed: 'H H(4-5) * * 1-5'