Add deb support to the libs (#109)

* Add deb support Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Rename deb Names Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> * Adding Deb test Signed-off-by: Peter Zhu <zhujiaxi@amazon.com> Signed-off-by: Peter Zhu <zhujiaxi@amazon.com>
opensearch-project · Jan 12, 2023 · ea7a7cc · ea7a7cc
1 parent b9588ac
commit ea7a7cc
Show file tree

Hide file tree

Showing 23 changed files with 5,455 additions and 28 deletions.
diff --git a/build.gradle b/build.gradle
@@ -120,7 +120,7 @@ jacocoTestReport {
     }
 }
 
-String version = '1.5.3'
+String version = '1.5.4'
 
 task updateVersion {
     doLast {

diff --git a/src/jenkins/BuildManifest.groovy b/src/jenkins/BuildManifest.groovy
@@ -42,6 +42,8 @@ class BuildManifest implements Serializable {
                 return 'zip'
             } else if (this.distribution == 'rpm') {
                 return 'rpm'
+            } else if (this.distribution == 'deb') {
+                return 'deb'
             } else {
                 return 'tar.gz'
             }

diff --git a/tests/data/opensearch-1.3.1-x64-deb.yml b/tests/data/opensearch-1.3.1-x64-deb.yml
@@ -0,0 +1,31 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch
+  version: 1.3.1
+  platform: linux
+  architecture: x64
+  distribution: deb
+  location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianle-opensearch-build-3-22/1.3.1/79/linux/x64/deb/deb/dist/opensearch/opensearch-1.3.1-linux-x64.deb
+  id: '79'
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: tags/1.3.1
+    commit_id: c4c0672877bf0f787ca857c7c37b775967f93d81
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianle-opensearch-build-3-22/1.3.1/79/linux/x64/deb/deb/builds/opensearch/dist/opensearch-min-1.3.1.x86_64.deb
+  - name: job-scheduler
+    repository: https://github.com/opensearch-project/job-scheduler.git
+    ref: tags/1.3.1.0
+    commit_id: 3a9b3447a6dea936cb72068fcd40381fb90bcfa6
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianle-opensearch-build-3-22/1.3.1/79/linux/x64/deb/deb/builds/opensearch/plugins/opensearch-job-scheduler-1.3.1.0.zip
+  - name: ml-commons
+    repository: https://github.com/opensearch-project/ml-commons.git
+    ref: tags/1.3.1.0
+    commit_id: 7f25eeb958bf01bf1c6b48ff5dc4e2e068c1d53b
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianle-opensearch-build-3-22/1.3.1/79/linux/x64/deb/deb/builds/opensearch/plugins/opensearch-ml-1.3.1.0.zip
+  - name: alerting
+    repository: https://github.com/opensearch-project/alerting.git
+    ref: tags/1.3.1.0
+    commit_id: cbbfd7419f79acdfddec156c83ed3ea8673a8166
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianle-opensearch-build-3-22/1.3.1/79/linux/x64/deb/deb/builds/opensearch/plugins/opensearch-alerting-1.3.1.0.zip
diff --git a/tests/data/opensearch-build-1.3.0-deb.yml b/tests/data/opensearch-build-1.3.0-deb.yml
diff --git a/tests/data/opensearch-build-2.0.0-rc1-deb.yml b/tests/data/opensearch-build-2.0.0-rc1-deb.yml
diff --git a/tests/data/opensearch-dashboards-1.3.0-x64-deb.yml b/tests/data/opensearch-dashboards-1.3.0-x64-deb.yml
@@ -0,0 +1,36 @@
+---
+schema-version: '1.1'
+build:
+  name: OpenSearch Dashboards
+  version: 1.3.0
+  platform: linux
+  architecture: x64
+  distribution: deb
+  location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/dist/opensearch-dashboards/opensearch-dashboards-1.3.0-linux-x64.deb
+  id: '24'
+components:
+  - name: OpenSearch-Dashboards
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    ref: tags/1.3.0
+    commit_id: 00e06934211e9819f99aabbf139885682f33e95e
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/builds/opensearch-dashboards/dist/opensearch-dashboards-min-1.3.0-x64.deb
+  - name: securityDashboards
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    ref: tags/1.3.0.0
+    commit_id: 10b4bb8edebd84c204e2c860ffbe8486f9510a8e
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/builds/opensearch-dashboards/plugins/securityDashboards-1.3.0.zip
+  - name: anomalyDetectionDashboards
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    ref: tags/1.3.0.0
+    commit_id: 5710e3dc6d654ed49d8012c318d9b21542e09487
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/builds/opensearch-dashboards/plugins/anomalyDetectionDashboards-1.3.0.zip
+  - name: reportsDashboards
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    ref: tags/1.3.0.0
+    commit_id: fec1eaf578f2b4ebbe95a972378b685eaf692def
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/builds/opensearch-dashboards/plugins/reportsDashboards-1.3.0.zip
+  - name: ganttChartDashboards
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    ref: tags/1.3.0.0
+    commit_id: 69fc0da7319db12ba3db81f3714ba81bddad6266
+    location: https://ci.opensearch.org/ci/dbc/Playground/tianleh-test/tianleh-osd-build-3-22/1.3.0/24/linux/x64/deb/deb/builds/opensearch-dashboards/plugins/ganttChartDashboards-1.3.0.zip
diff --git a/tests/data/opensearch-dashboards-build-1.3.0-deb.yml b/tests/data/opensearch-dashboards-build-1.3.0-deb.yml
@@ -0,0 +1,87 @@
+---
+build:
+  architecture: x64
+  id: 25b38c278cdd45efa583765d8ba76346
+  name: OpenSearch Dashboards
+  platform: linux
+  version: 1.3.0
+  distribution: deb
+components:
+  - artifacts:
+      dist:
+        - dist/opensearch-dashboards-min-1.3.0-linux-x64.deb
+    commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
+    name: OpenSearch-Dashboards
+    ref: tags/1.3.0
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/alertingDashboards-1.3.0.zip
+    commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
+    name: alertingDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin
+    version: 1.3.0.0
+  - commit_id: 208227256bf6dece2976da528bf41bae56e4effc
+    name: functionalTestDashboards
+    ref: main
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/securityDashboards-1.3.0.zip
+    commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
+    name: securityDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/indexManagementDashboards-1.3.0.zip
+    commit_id: 6748b18ee172138b4811a459053f4a43c1486821
+    name: indexManagementDashboards
+    ref: '1.3'
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/queryWorkbenchDashboards-1.3.0.zip
+    commit_id: a675eac2ed32a228184c47c289cfd22fde479139
+    name: queryWorkbenchDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/sql.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/reportsDashboards-1.3.0.zip
+    commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
+    name: reportsDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/observabilityDashboards-1.3.0.zip
+    commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
+    name: observabilityDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/trace-analytics.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/ganttChartDashboards-1.3.0.zip
+    commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
+    name: ganttChartDashboards
+    ref: 1.3.0.0
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    version: 1.3.0.0
+  - artifacts:
+      plugins:
+        - plugins/anomalyDetectionDashboards-1.3.0.zip
+    commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
+    name: anomalyDetectionDashboards
+    ref: tags/1.3.0.0
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    version: 1.3.0.0
+schema-version: '1.2'
diff --git a/tests/data/opensearch-dashboards-build-2.0.0-rc1-deb.yml b/tests/data/opensearch-dashboards-build-2.0.0-rc1-deb.yml
@@ -0,0 +1,87 @@
+---
+build:
+  architecture: x64
+  id: 25b38c278cdd45efa583765d8ba76346
+  name: OpenSearch Dashboards
+  platform: linux
+  version: 2.0.0-rc1
+  distribution: deb
+components:
+  - artifacts:
+      dist:
+        - dist/opensearch-dashboards-min-2.0.0-rc1-linux-x64.deb
+    commit_id: caf668e73304bac890f41c37cd6c3a41257cd289
+    name: OpenSearch-Dashboards
+    ref: tags/2.0.0-rc1
+    repository: https://github.com/opensearch-project/OpenSearch-Dashboards.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/alertingDashboards-2.0.0-rc1.zip
+    commit_id: 07d4cb8bc9c62b15bf7cb4fd4e3fc3a818f9152d
+    name: alertingDashboards
+    ref: tags/2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/alerting-dashboards-plugin
+    version: 2.0.0.0-rc1
+  - commit_id: 208227256bf6dece2976da528bf41bae56e4effc
+    name: functionalTestDashboards
+    ref: main
+    repository: https://github.com/opensearch-project/opensearch-dashboards-functional-test.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/securityDashboards-2.0.0-rc1.zip
+    commit_id: 960ef753f74dceb793e8936a40ac8c6cbec88dd1
+    name: securityDashboards
+    ref: '2.0'
+    repository: https://github.com/opensearch-project/security-dashboards-plugin.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/indexManagementDashboards-2.0.0-rc1.zip
+    commit_id: 6748b18ee172138b4811a459053f4a43c1486821
+    name: indexManagementDashboards
+    ref: '2.0'
+    repository: https://github.com/opensearch-project/index-management-dashboards-plugin
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/queryWorkbenchDashboards-2.0.0-rc1.zip
+    commit_id: a675eac2ed32a228184c47c289cfd22fde479139
+    name: queryWorkbenchDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/sql.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/reportsDashboards-2.0.0-rc1.zip
+    commit_id: 290523a6d825c9c073818aebfcbe4813cb2554bc
+    name: reportsDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/dashboards-reports.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/observabilityDashboards-2.0.0-rc1.zip
+    commit_id: 4fde4040c0a43233b41a862a169e2bd1616f6b1b
+    name: observabilityDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/trace-analytics.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/ganttChartDashboards-2.0.0-rc1.zip
+    commit_id: 2123bc65873e026cb8b0fd26816a6b1980bd19ce
+    name: ganttChartDashboards
+    ref: 2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/dashboards-visualizations.git
+    version: 2.0.0.0-rc1
+  - artifacts:
+      plugins:
+        - plugins/anomalyDetectionDashboards-2.0.0-rc1.zip
+    commit_id: c4921f4471aeab38db80eb47cd92b4f7a3e04126
+    name: anomalyDetectionDashboards
+    ref: tags/2.0.0.0-rc1
+    repository: https://github.com/opensearch-project/anomaly-detection-dashboards-plugin
+    version: 2.0.0.0-rc1
+schema-version: '1.2'
diff --git a/tests/jenkins/TestPromoteLinuxArtifacts.groovy b/tests/jenkins/TestPromoteLinuxArtifacts.groovy
@@ -22,6 +22,10 @@ class TestPromoteLinuxArtifacts extends BuildPipelineTest {
     private Path targetOpenSearchDashboardsRpm;
     private Path targetOpenSearchRpmQualifier;
     private Path targetOpenSearchDashboardsRpmQualifier;
+    private Path targetOpenSearchDeb;
+    private Path targetOpenSearchDashboardsDeb;
+    private Path targetOpenSearchDebQualifier;
+    private Path targetOpenSearchDashboardsDebQualifier;
 
     @Override
     @Before
@@ -118,6 +122,26 @@ class TestPromoteLinuxArtifacts extends BuildPipelineTest {
             "tests/data/opensearch-dashboards-build-2.0.0-rc1-rpm.yml", 
             "tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/builds/opensearch-dashboards/manifest.yml"
         );
+
+        targetOpenSearchDeb = copy(
+            "tests/data/opensearch-build-1.3.0-deb.yml", 
+            "tests/jenkins/artifacts/deb/vars-build/1.3.0/33/linux/x64/deb/builds/opensearch/manifest.yml"
+        );
+
+        targetOpenSearchDashboardsDeb = copy(
+            "tests/data/opensearch-dashboards-build-1.3.0-deb.yml", 
+            "tests/jenkins/artifacts/deb/vars-build/1.3.0/33/linux/x64/deb/builds/opensearch-dashboards/manifest.yml"
+        );
+
+        targetOpenSearchDebQualifier = copy(
+            "tests/data/opensearch-build-2.0.0-rc1-deb.yml", 
+            "tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch/manifest.yml"
+        );
+
+        targetOpenSearchDashboardsDebQualifier = copy(
+            "tests/data/opensearch-dashboards-build-2.0.0-rc1-deb.yml", 
+            "tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch-dashboards/manifest.yml"
+        );
     }
 
     private Path copy(String sourcePath, String targetPath){
@@ -141,6 +165,10 @@ class TestPromoteLinuxArtifacts extends BuildPipelineTest {
         Files.delete(targetOpenSearchDashboardsRpm)
         Files.delete(targetOpenSearchRpmQualifier)
         Files.delete(targetOpenSearchDashboardsRpmQualifier)
+        Files.delete(targetOpenSearchDeb)
+        Files.delete(targetOpenSearchDashboardsDeb)
+        Files.delete(targetOpenSearchDebQualifier)
+        Files.delete(targetOpenSearchDashboardsDebQualifier)
     }
 
     @Test

diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_Jenkinsfile.txt
@@ -33,3 +33,12 @@
                      promoteArtifacts.println(Signing Core/Bundle Artifacts)
                      promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                         promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*})
+                     promoteArtifacts.println(S3 download linux x64 deb 2.0.0-rc1 from vars-build build number 33 artifacts before creating signatures)
+                     promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/deb, path=vars-build/2.0.0-rc1/33/linux/x64/deb/, force=true})
+                     promoteArtifacts.readYaml({file=tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch/manifest.yml})
+                     promoteArtifacts.fileExists(tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch/core-plugins)
+                     promoteArtifacts.println(Signing Starts)
+                     promoteArtifacts.println(Signing Core/Bundle Artifacts)
+                     promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/dist/opensearch/, includePathPattern=**/opensearch-2.0.0-rc1-linux-x64*})
diff --git a/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt b/tests/jenkins/jobs/PromoteArtifactsQualifier_OpenSearch_Dashboards_Jenkinsfile.txt
@@ -33,3 +33,12 @@
                      promoteArtifacts.println(Signing Core/Bundle Artifacts)
                      promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
                         promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/rpm/vars-build/2.0.0-rc1/33/linux/x64/rpm/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*})
+                     promoteArtifacts.println(S3 download linux x64 deb 2.0.0-rc1 from vars-build build number 33 artifacts before creating signatures)
+                     promoteArtifacts.withAWS({role=opensearch-bundle, roleAccount=AWS_ACCOUNT_PUBLIC, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteArtifacts.s3Download({bucket=ARTIFACT_BUCKET_NAME, file=tests/jenkins/artifacts/deb, path=vars-build/2.0.0-rc1/33/linux/x64/deb/, force=true})
+                     promoteArtifacts.readYaml({file=tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch-dashboards/manifest.yml})
+                     promoteArtifacts.fileExists(tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/builds/opensearch-dashboards/core-plugins)
+                     promoteArtifacts.println(Signing Starts)
+                     promoteArtifacts.println(Signing Core/Bundle Artifacts)
+                     promoteArtifacts.withAWS({role=ARTIFACT_PROMOTION_ROLE_NAME, roleAccount=AWS_ACCOUNT_ARTIFACT, duration=900, roleSessionName=jenkins-session}, groovy.lang.Closure)
+                        promoteArtifacts.s3Upload({bucket=ARTIFACT_PRODUCTION_BUCKET_NAME, path=releases/bundle/opensearch-dashboards/2.0.0-rc1/, workingDir=tests/jenkins/artifacts/deb/vars-build/2.0.0-rc1/33/linux/x64/deb/dist/opensearch-dashboards/, includePathPattern=**/opensearch-dashboards-2.0.0-rc1-linux-x64*})