ci: stash/unstash steps with Google Cloud Storage

local/configs/jenkins.yaml

@@ -93,7 +93,6 @@ unclassified:
                 filterSettings:
                   includeFilter: false
                   selectors:
-                  - wildcard: "*.*"
                   - wildcard: ".ci/**"
                   - wildcard: ".mvn/**"
                   - wildcard: "local/**"
@@ -128,6 +127,7 @@ jobs:
   - file: "/var/pipeline-library/src/test/resources/jobs/parentstream.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/pipelineManager.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/rebuildPipeline.dsl"
+  - file: "/var/pipeline-library/src/test/resources/jobs/stashV2.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/timeout/downstream.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/timeout/parentstream.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/withGitRelease.dsl"

src/test/groovy/ApmBasePipelineTest.groovy

@@ -271,6 +271,8 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
         throw new Exception('Failed')
       }
     })
+    helper.registerAllowedMethod('googleStorageDownload', [Map.class], {true})
+    helper.registerAllowedMethod('googleStorageUpload', [Map.class], {true})
     helper.registerAllowedMethod('isUnix', [ ], { true })
     helper.registerAllowedMethod('junit', [Map.class], null)
     helper.registerAllowedMethod('lastWithArtifacts', [ ], null)

src/test/groovy/StashV2StepTests.groovy

@@ -0,0 +1,143 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertNull
+import static org.junit.Assert.assertTrue
+
+class StashV2StepTests extends ApmBasePipelineTest {
+  String scriptName = 'vars/stashV2.groovy'
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+  }
+
+  @Test
+  void test_without_name_param() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call()
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'stashV2: name param is required.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_bucket_param_without_env_variable() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call(name: 'source')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'stashV2: bucket param is required or JOB_GCS_BUCKET env variable.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_credentialsId_param_without_env_variable() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call(name: 'source', bucket: 'foo')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'stashV2: credentialsId param is required or JOB_GCS_CREDENTIALS env variable.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_bucket_precedence() throws Exception {
+    def script = loadScript(scriptName)
+    env.JOB_GCS_BUCKET = 'bar'
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_BUCKET is set. bucket param got precedency instead.'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=secret'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_credentialsId_precedence() throws Exception {
+    def script = loadScript(scriptName)
+    env.JOB_GCS_CREDENTIALS = 'bar'
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'my-super-credentials')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_CREDENTIALS is set. credentialsId param got precedency instead.'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=my-super-credentials'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_linux_with_parameters() throws Exception {
+    def script = loadScript(scriptName)
+    def bucketUri = script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
+    assertNull(assertMethodCall('bat'))
+    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertTrue(bucketUri.contains('source/source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_windows_with_parameters() throws Exception {
+    def script = loadScript(scriptName)
+    helper.registerAllowedMethod('isUnix', [], { false })
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
+    assertNull(assertMethodCall('sh'))
+    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'tar --exclude=source.tgz -czf source.tgz .'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'del source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_missing_build_context() throws Exception {
+    def script = loadScript(scriptName)
+    env.remove('BUILD_ID')
+    try {
+      script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'BUILD_ID and JOB_NAME environment variables are required.'))
+    assertJobStatusFailure()
+  }
+}

src/test/groovy/UnstashV2StepTests.groovy

@@ -0,0 +1,140 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Before
+import org.junit.Test
+import static org.junit.Assert.assertFalse
+import static org.junit.Assert.assertNull
+import static org.junit.Assert.assertTrue
+
+class UnstashV2StepTests extends ApmBasePipelineTest {
+  String scriptName = 'vars/unstashV2.groovy'
+
+  @Override
+  @Before
+  void setUp() throws Exception {
+    super.setUp()
+  }
+
+  @Test
+  void test_without_name_param() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call()
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'unstashV2: name param is required.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_bucket_param_without_env_variable() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call(name: 'source')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'unstashV2: bucket param is required or JOB_GCS_BUCKET env variable.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_without_credentialsId_param_without_env_variable() throws Exception {
+    def script = loadScript(scriptName)
+    try {
+      script.call(name: 'source', bucket: 'foo')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'unstashV2: credentialsId param is required or JOB_GCS_CREDENTIALS env variable.'))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_bucket_precedence() throws Exception {
+    def script = loadScript(scriptName)
+    env.JOB_GCS_BUCKET = 'bar'
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('log', 'unstashV2: JOB_GCS_BUCKET is set. bucket param got precedency instead.'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar -xpf source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'bucketUri=gs://foo'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'credentialsId=secret'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'source/source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_credentialsId_precedence() throws Exception {
+    def script = loadScript(scriptName)
+    env.JOB_GCS_CREDENTIALS = 'bar'
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'my-super-credentials')
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('log', 'unstashV2: JOB_GCS_CREDENTIALS is set. credentialsId param got precedency instead.'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar -xpf source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'bucketUri=gs://foo'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'credentialsId=my-super-credentials'))
+    assertTrue(assertMethodCallContainsPattern('googleStorageDownload', 'source/source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_linux_with_parameters() throws Exception {
+    def script = loadScript(scriptName)
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
+    assertNull(assertMethodCall('bat'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar -xpf source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_windows_with_parameters() throws Exception {
+    def script = loadScript(scriptName)
+    helper.registerAllowedMethod('isUnix', [], { false })
+    script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    printCallStack()
+    assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
+    assertNull(assertMethodCall('sh'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'tar -xpf source.tgz'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'del source.tgz'))
+    assertJobStatusSuccess()
+  }
+
+  @Test
+  void test_missing_build_context() throws Exception {
+    def script = loadScript(scriptName)
+    env.remove('BUILD_ID')
+    try {
+      script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
+    } catch(e) {
+      //NOOP
+    }
+    printCallStack()
+    assertTrue(assertMethodCallContainsPattern('error', 'BUILD_ID and JOB_NAME environment variables are required.'))
+    assertJobStatusFailure()
+  }
+}

src/test/resources/jobs/stashV2.dsl

@@ -0,0 +1,44 @@
+NAME = 'it/stashV2'
+DSL = '''pipeline {
+  agent any
+  stages {
+    stage('stashV2') {
+      steps {
+        // Credentials for the Google Storage are file based, so it's not
+        // fully automated yet, for such you need to create a new credentials
+        // manually
+        input 'Please create Google Service account from private key  http://localhost:18080/credentials/store/system/domain/_/newCredentials'
+        gitCheckout(basedir: 'sub-folder', branch: 'master',
+                    credentialsId: '2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken',
+                    repo: 'https://github.com/octocat/Hello-World.git')
+        withEnv(["JOB_GCS_BUCKET=beats-ci-temp", "JOB_GCS_CREDENTIALS=beats-ci-gcs-plugin"]){
+          stashV2(name: 'source')
+        }
+        stashV2(name: 'foo', bucket: 'beats-ci-temp', credentialsId: 'beats-ci-gcs-plugin')
+        deleteDir()
+      }
+    }
+    stage('unstashV2'){
+      steps {
+        dir('source') {
+          withEnv(['JOB_GCS_BUCKET=beats-ci-temp', 'JOB_GCS_CREDENTIALS=beats-ci-gcs-plugin']){
+            unstashV2(name: 'source')
+          }
+          sh 'ls -ltra sub-folder'
+        }
+        dir('foo') {
+          unstashV2(name: 'foo', bucket: 'beats-ci-temp', credentialsId: 'beats-ci-gcs-plugin')
+          sh 'ls -ltra sub-folder'
+        }
+      }
+    }
+  }
+}'''
+
+pipelineJob(NAME) {
+  definition {
+    cps {
+      script(DSL.stripIndent())
+    }
+  }
+}

test-infra/apm-ci/test_installed_tools.py

@@ -21,6 +21,10 @@ def test_python_installed(host):
   cmd = host.run("python --version")
   assert cmd.rc == 0, "it is required for the apm-agent-python and apm-integration-testing"
 
+def test_tar_installed(host):
+  cmd = host.run("tar --version")
+  assert cmd.rc == 0, "it is required for the stashV2 and unstashV2 steps"
+
 def test_vault_installed(host):
   cmd = host.run("vault --version")
   assert cmd.rc == 0, "it is required for all the APM projects"

test-infra/apm-ci/test_windows.py

@@ -7,3 +7,7 @@ def test_dotnet_installed(host):
 def test_python_installed(host):
   cmd = host.run("python --version")
   assert cmd.rc == 0, "it is required for the apm-agent-python"
+
+def test_tar_installed(host):
+  cmd = host.run("tar --version")
+  assert cmd.rc == 0, "it is required for the stashV2 and unstashV2 steps"