Skip to content

Commit

Permalink
Add windows zip support for promotion artifacts (#31)
Browse files Browse the repository at this point in the history
* Add windows zip support for promotion artifacts

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Update tests

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

* Update test cases

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>

Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
  • Loading branch information
peterzhuamazon authored Nov 9, 2022
1 parent 8c4a662 commit acd16a6
Show file tree
Hide file tree
Showing 25 changed files with 5,657 additions and 20 deletions.
2,486 changes: 2,486 additions & 0 deletions tests/data/opensearch-build-1.3.0-windows-zip.yml

Large diffs are not rendered by default.

2,486 changes: 2,486 additions & 0 deletions tests/data/opensearch-build-2.0.0-rc1-windows-zip.yml

Large diffs are not rendered by default.

87 changes: 87 additions & 0 deletions tests/data/opensearch-dashboards-build-1.3.0-windows-zip.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
---
build:
architecture: x64
id: 25b38c278cdd45efa583765d8ba76346
name: OpenSearch Dashboards
platform: windows
version: 1.3.0
distribution: zip
components:
- artifacts:
dist:
- dist/opensearch-dashboards-min-1.3.0-windows-x64.zip
commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
name: OpenSearch-Dashboards
ref: tags/1.3.0
repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/alertingDashboards-1.3.0.zip
commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
name: alertingDashboards
ref: tags/1.3.0.0
repository: https://github.com/opensearch-project/alerting-dashboards-plugin
version: 1.3.0.0
- commit_id: 208227256bf6dece2976da528bf41bae56e4effc
name: functionalTestDashboards
ref: main
repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/securityDashboards-1.3.0.zip
commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
name: securityDashboards
ref: '1.3'
repository: https://github.com/opensearch-project/security-dashboards-plugin.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/indexManagementDashboards-1.3.0.zip
commit_id: 6748b18ee172138b4811a459053f4a43c1486821
name: indexManagementDashboards
ref: '1.3'
repository: https://github.com/opensearch-project/index-management-dashboards-plugin
version: 1.3.0.0
- artifacts:
plugins:
- plugins/queryWorkbenchDashboards-1.3.0.zip
commit_id: a675eac2ed32a228184c47c289cfd22fde479139
name: queryWorkbenchDashboards
ref: 1.3.0.0
repository: https://github.com/opensearch-project/sql.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/reportsDashboards-1.3.0.zip
commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
name: reportsDashboards
ref: 1.3.0.0
repository: https://github.com/opensearch-project/dashboards-reports.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/observabilityDashboards-1.3.0.zip
commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
name: observabilityDashboards
ref: 1.3.0.0
repository: https://github.com/opensearch-project/trace-analytics.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/ganttChartDashboards-1.3.0.zip
commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
name: ganttChartDashboards
ref: 1.3.0.0
repository: https://github.com/opensearch-project/dashboards-visualizations.git
version: 1.3.0.0
- artifacts:
plugins:
- plugins/anomalyDetectionDashboards-1.3.0.zip
commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
name: anomalyDetectionDashboards
ref: tags/1.3.0.0
repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
version: 1.3.0.0
schema-version: '1.2'
87 changes: 87 additions & 0 deletions tests/data/opensearch-dashboards-build-2.0.0-rc1-windows-zip.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
---
build:
architecture: x64
id: 25b38c278cdd45efa583765d8ba76346
name: OpenSearch Dashboards
platform: windows
version: 2.0.0-rc1
distribution: zip
components:
- artifacts:
dist:
- dist/opensearch-dashboards-min-2.0.0-rc1-windows-x64.zip
commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
name: OpenSearch-Dashboards
ref: tags/2.0.0-rc1
repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/alertingDashboards-2.0.0-rc1.zip
commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
name: alertingDashboards
ref: tags/2.0.0.0-rc1
repository: https://github.com/opensearch-project/alerting-dashboards-plugin
version: 2.0.0.0-rc1
- commit_id: 208227256bf6dece2976da528bf41bae56e4effc
name: functionalTestDashboards
ref: main
repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/securityDashboards-2.0.0-rc1.zip
commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
name: securityDashboards
ref: '2.0'
repository: https://github.com/opensearch-project/security-dashboards-plugin.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/indexManagementDashboards-2.0.0-rc1.zip
commit_id: 6748b18ee172138b4811a459053f4a43c1486821
name: indexManagementDashboards
ref: '2.0'
repository: https://github.com/opensearch-project/index-management-dashboards-plugin
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/queryWorkbenchDashboards-2.0.0-rc1.zip
commit_id: a675eac2ed32a228184c47c289cfd22fde479139
name: queryWorkbenchDashboards
ref: 2.0.0.0-rc1
repository: https://github.com/opensearch-project/sql.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/reportsDashboards-2.0.0-rc1.zip
commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
name: reportsDashboards
ref: 2.0.0.0-rc1
repository: https://github.com/opensearch-project/dashboards-reports.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/observabilityDashboards-2.0.0-rc1.zip
commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
name: observabilityDashboards
ref: 2.0.0.0-rc1
repository: https://github.com/opensearch-project/trace-analytics.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/ganttChartDashboards-2.0.0-rc1.zip
commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
name: ganttChartDashboards
ref: 2.0.0.0-rc1
repository: https://github.com/opensearch-project/dashboards-visualizations.git
version: 2.0.0.0-rc1
- artifacts:
plugins:
- plugins/anomalyDetectionDashboards-2.0.0-rc1.zip
commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
name: anomalyDetectionDashboards
ref: tags/2.0.0.0-rc1
repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
version: 2.0.0.0-rc1
schema-version: '1.2'
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import org.junit.*
import java.util.*
import java.nio.file.*

class TestPromoteArtifacts extends BuildPipelineTest {
class TestPromoteLinuxArtifacts extends BuildPipelineTest {
private Path targetOpenSearchTar;
private Path targetOpenSearchDashboardsTar;
private Path targetOpenSearchTarQualifier;
Expand Down
156 changes: 156 additions & 0 deletions tests/jenkins/TestPromoteWindowsArtifacts.groovy
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package jenkins.tests

import org.junit.*
import java.util.*
import java.nio.file.*

class TestPromoteWindowsArtifacts extends BuildPipelineTest {
private Path targetOpenSearchWindowsZip;
private Path targetOpenSearchDashboardsWindowsZip;
private Path targetOpenSearchWindowsZipQualifier;
private Path targetOpenSearchDashboardsWindowsZipQualifier;

@Override
@Before
void setUp() {
super.setUp()

binding.setVariable('PUBLIC_ARTIFACT_URL', 'https://ci.opensearch.org/dbc')
binding.setVariable('DISTRIBUTION_JOB_NAME', 'vars-build')
binding.setVariable('STAGE_NAME', 'stage')
binding.setVariable('BUILD_URL', 'http://jenkins.us-east-1.elb.amazonaws.com/job/vars/42')
binding.setVariable('DISTRIBUTION_BUILD_NUMBER', '33')
binding.setVariable('DISTRIBUTION_PLATFORM', 'windows')
binding.setVariable('DISTRIBUTION_ARCHITECTURE', 'x64')
binding.setVariable('WORKSPACE', 'tests/jenkins')
binding.setVariable('GITHUB_BOT_TOKEN_NAME', 'github_bot_token_name')
def configs = ["role": "dummy_role",
"external_id": "dummy_ID",
"unsigned_bucket": "dummy_unsigned_bucket",
"signed_bucket": "dummy_signed_bucket"]
binding.setVariable('configs', configs)
helper.registerAllowedMethod("readJSON", [Map.class], {c -> configs})

helper.registerAllowedMethod("git", [Map])
helper.registerAllowedMethod("s3Download", [Map])
helper.registerAllowedMethod("s3Upload", [Map])
helper.registerAllowedMethod("withCredentials", [Map, Closure], { args, closure ->
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.registerAllowedMethod("withAWS", [Map, Closure], { args, closure ->
closure.delegate = delegate
return helper.callClosure(closure)
})
helper.registerAllowedMethod('getPath', { args ->
return "tests/jenkins/file/found.zip"
})
helper.registerAllowedMethod('findFiles', [Map], { args ->
return [{}]
})
helper.addFileExistsMock('tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch/core-plugins', true)

helper.addShMock('find tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch/core-plugins -type f') { script ->
return [stdout: "zip_dummy_artifact_1.3.0.zip dummy_artifact_1.3.0.dummy", exitValue: 0]
}
helper.addShMock('sha512sum zip_dummy_artifact_1.3.0.zip') { script ->
return [stdout: "shaHashDummy_zip_dummy_artifact_1.3.0.zip tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch/core-plugins/zip_dummy_artifact_1.3.0.zip", exitValue: 0]
}
helper.addShMock('basename tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch/core-plugins/zip_dummy_artifact_1.3.0.zip') { script ->
return [stdout: "zip_dummy_artifact_1.3.0.zip", exitValue: 0]
}

targetOpenSearchWindowsZip = copy(
"tests/data/opensearch-build-1.3.0-windows-zip.yml",
"tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch/manifest.yml"
);
targetOpenSearchDashboardsWindowsZip = copy(
"tests/data/opensearch-dashboards-build-1.3.0-windows-zip.yml",
"tests/jenkins/artifacts/zip/vars-build/1.3.0/33/windows/x64/zip/builds/opensearch-dashboards/manifest.yml"
);
targetOpenSearchWindowsZipQualifier = copy(
"tests/data/opensearch-build-2.0.0-rc1-windows-zip.yml",
"tests/jenkins/artifacts/zip/vars-build/2.0.0-rc1/33/windows/x64/zip/builds/opensearch/manifest.yml"
);
targetOpenSearchDashboardsWindowsZipQualifier = copy(
"tests/data/opensearch-dashboards-build-2.0.0-rc1-windows-zip.yml",
"tests/jenkins/artifacts/zip/vars-build/2.0.0-rc1/33/windows/x64/zip/builds/opensearch-dashboards/manifest.yml"
);
}

private Path copy(String sourcePath, String targetPath){
Path source = Path.of(sourcePath);
Path target = Path.of(targetPath);
Files.createDirectories(target.getParent());
Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);

return target;
}

@After
void after() {
super.setUp()
// Test file needs to be cleaned up
Files.delete(targetOpenSearchWindowsZip)
Files.delete(targetOpenSearchDashboardsWindowsZip)
Files.delete(targetOpenSearchWindowsZipQualifier)
Files.delete(targetOpenSearchDashboardsWindowsZipQualifier)
}

@Test
public void testDefault() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifacts_Jenkinsfile_Windows.txt")
}

@Test
public void testDefault_OpenSearch_Dashboards() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifacts_OpenSearch_Dashboards_Jenkinsfile_Windows.txt")
}

@Test
public void testDefaultQualifier() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile_Windows.txt")
}

@Test
public void testDefaultQualifier_OpenSearch_Dashboards() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile_Windows.txt")
}

@Test
public void testWithActions() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifacts_actions_Jenkinsfile_Windows.txt")
}

@Test
public void testWithActions_OpenSearch_Dashboards() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifacts_actions_OpenSearch_Dashboards_Jenkinsfile_Windows.txt")
}

@Test
public void testWithActionsQualifier() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifactsQualifier_actions_Jenkinsfile_Windows.txt")
}

@Test
public void testWithActionsQualifier_OpenSearch_Dashboards() {
super.testPipeline("tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile",
"tests/jenkins/jobs/PromoteArtifactsQualifier_actions_OpenSearch_Dashboards_Jenkinsfile_Windows.txt")
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This page intentionally left blank.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
This page intentionally left blank.
4 changes: 2 additions & 2 deletions tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteArtifacts.println(S3 download tar artifacts before creating signatures)
promoteArtifacts.println(S3 download linux x64 tar 2.0.0-rc1 from vars-build build number 33 artifacts before creating signatures)
promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/tar, path=vars-build/2.0.0-rc1/33/linux/x64/tar/, force=true})
promoteArtifacts.readYaml({file=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/manifest.yml})
Expand All @@ -24,7 +24,7 @@
promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/core/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/builds/opensearch/dist/, includePathPattern=**/opensearch-min-2.0.0-rc1-linux-x64*})
promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/tar/vars-build/2.0.0-rc1/33/linux/x64/tar/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*})
promoteArtifacts.println(S3 download rpm artifacts before creating signatures)
promoteArtifacts.println(S3 download linux x64 rpm 2.0.0-rc1 from vars-build build number 33 artifacts before creating signatures)
promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/rpm, path=vars-build/2.0.0-rc1/33/linux/x64/rpm/, force=true})
promoteArtifacts.readYaml({file=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch/manifest.yml})
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
PromoteArtifactsQualifier_Jenkinsfile.run()
PromoteArtifactsQualifier_Jenkinsfile.pipeline(groovy.lang.Closure)
PromoteArtifactsQualifier_Jenkinsfile.echo(Executing on agent [label:none])
PromoteArtifactsQualifier_Jenkinsfile.stage(promote, groovy.lang.Closure)
PromoteArtifactsQualifier_Jenkinsfile.script(groovy.lang.Closure)
PromoteArtifactsQualifier_Jenkinsfile.promoteArtifacts({manifest=tests/data/opensearch-2.0.0-rc1.yml})
promoteArtifacts.legacySCM(groovy.lang.Closure)
promoteArtifacts.library({identifier=jenkins@main, retriever=null})
promoteArtifacts.readYaml({file=tests/data/opensearch-2.0.0-rc1.yml})
InputManifest.asBoolean()
promoteArtifacts.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC})
promoteArtifacts.string({credentialsId=jenkins-artifact-bucket-name, variable=ARTIFACT_BUCKET_NAME})
promoteArtifacts.string({credentialsId=jenkins-artifact-promotion-role, variable=ARTIFACT_PROMOTION_ROLE_NAME})
promoteArtifacts.string({credentialsId=jenkins-aws-production-account, variable=AWS_ACCOUNT_ARTIFACT})
promoteArtifacts.string({credentialsId=jenkins-artifact-production-bucket-name, variable=ARTIFACT_PRODUCTION_BUCKET_NAME})
promoteArtifacts.withCredentials([AWS_ACCOUNT_PUBLIC, ARTIFACT_BUCKET_NAME, ARTIFACT_PROMOTION_ROLE_NAME, AWS_ACCOUNT_ARTIFACT, ARTIFACT_PRODUCTION_BUCKET_NAME], groovy.lang.Closure)
promoteArtifacts.println(S3 download windows x64 zip 2.0.0-rc1 from vars-build build number 33 artifacts before creating signatures)
promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/zip, path=vars-build/2.0.0-rc1/33/windows/x64/zip/, force=true})
promoteArtifacts.readYaml({file=tests/jenkins/artifacts/zip/vars-build/2.0.0-rc1/33/windows/x64/zip/builds/opensearch/manifest.yml})
promoteArtifacts.fileExists(tests/jenkins/artifacts/zip/vars-build/2.0.0-rc1/33/windows/x64/zip/builds/opensearch/core-plugins)
promoteArtifacts.println(Signing Starts)
promoteArtifacts.println(Signing Core/Bundle Artifacts)
promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/zip/vars-build/2.0.0-rc1/33/windows/x64/zip/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-windows-x64*})
Loading

0 comments on commit acd16a6

Please sign in to comment.