diff --git a/jenkins/opensearch/distribution-build.jenkinsfile b/jenkins/opensearch/distribution-build.jenkinsfile index 33584a4b30..5e33eb55cf 100644 --- a/jenkins/opensearch/distribution-build.jenkinsfile +++ b/jenkins/opensearch/distribution-build.jenkinsfile @@ -41,6 +41,12 @@ pipeline { defaultValue: "bwc-test", trim: true ) + string( // Note: need to update 'verify-parameters' entries if you add new platform(s) + name: 'BUILD_PLATFORM', + description: "Build selected platform related artifacts, choices include 'linux', 'macos', 'windows'. Can combine multiple platforms with space in between (maven snapshot is only on linux)", + defaultValue: 'linux macos windows', + trim: true + ) choice( name: 'BUILD_DOCKER', description: 'Build docker image or not with options.', @@ -58,7 +64,7 @@ pipeline { ) } stages { - stage('detect docker image + args') { + stage('verify-parameters') { agent { docker { label AGENT_X64 @@ -68,8 +74,21 @@ pipeline { } steps { script { + echo("Detect Docker Images and Related Parameters") dockerAgent = detectDockerAgent() currentBuild.description = INPUT_MANIFEST + + echo("Verify Build Platforms") + def build_platform_array = params.BUILD_PLATFORM.tokenize(' ') + echo("User Entry Platforms: '${params.BUILD_PLATFORM}', ${build_platform_array}") + def all_platforms = "linux macos windows" + echo("All Supported Platforms: '${all_platforms}'") + for (String plat : build_platform_array) { + if (! all_platforms.contains(plat)) { + currentBuild.result = 'ABORTED' + error("Missing parameter: BUILD_PLATFORM (possible entries: ${all_platforms}).") + } + } } } } @@ -99,7 +118,13 @@ pipeline { } stage('build') { parallel { - stage('build-snapshot-x64-tar') { + stage('build-snapshot-linux-x64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } environment { SNAPSHOT_REPO_URL = "https://aws.oss.sonatype.org/content/repositories/snapshots/" } @@ -116,6 +141,8 @@ pipeline { buildManifest( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'linux', + architecture: 'x64', distribution: 'tar', snapshot: true ) @@ -142,7 +169,13 @@ pipeline { } } } - stage('build-snapshot-arm64-tar') { + stage('build-snapshot-linux-arm64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { docker { label AGENT_ARM64 @@ -156,6 +189,8 @@ pipeline { buildManifest( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'linux', + architecture: 'x64', distribution: 'tar', snapshot: true ) @@ -173,6 +208,12 @@ pipeline { } } stage('build-snapshot-macos-x64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('macos') + } + } agent { node { label 'Jenkins-Agent-MacOS12-X64-Mac1Metal-Multi-Host' @@ -186,6 +227,8 @@ pipeline { buildManifest( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'darwin', + architecture: 'x64', distribution: 'tar', snapshot: true ) @@ -202,38 +245,51 @@ pipeline { } } } - // stage('build-snapshot-windows-x64-zip') { - // agent { - // node { - // label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' - // } - // } - // tools { - // jdk dockerAgent.javaVersion - // } - // steps { - // script { - // buildManifest( - // componentName: "${COMPONENT_NAME}", - // inputManifest: "manifests/${INPUT_MANIFEST}", - // distribution: 'zip', - // platform: 'windows', - // snapshot: true - // ) - // echo("Uploading windows min snapshots to S3") - // uploadMinSnapshotsToS3( - // fileActions: [createSha512Checksums()], - // distribution: 'zip' - // ) - // } - // } - // post { - // always { - // postCleanup() - // } - // } - // } - stage('build-and-test-x64-tar') { + stage('build-snapshot-windows-x64-zip') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('windows') + } + } + agent { + node { + label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' + } + } + tools { + jdk dockerAgent.javaVersion + } + steps { + script { + buildManifest( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'windows', + architecture: 'x64', + distribution: 'zip', + snapshot: true + ) + echo("Uploading windows min snapshots to S3") + uploadMinSnapshotsToS3( + fileActions: [createSha512Checksums()], + distribution: 'zip' + ) + } + } + post { + always { + postCleanup() + } + } + } + stage('build-and-test-linux-x64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { docker { label AGENT_X64 @@ -247,15 +303,17 @@ pipeline { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'linux', + architecture: 'x64', distribution: "tar" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_X64_TAR = artifactUrl + env.ARTIFACT_URL_LINUX_X64_TAR = artifactUrl env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") - echo "buildManifestUrl (x64, tar): ${buildManifestUrl}" - echo "artifactUrl (x64, tar): ${artifactUrl}" + echo "buildManifestUrl (linux, x64, tar): ${buildManifestUrl}" + echo "artifactUrl (linux, x64, tar): ${artifactUrl}" parallel([ 'integ-test': { @@ -275,11 +333,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: integTestResults.getId(), - stage: "integ_test_x64" + stage: "integ_test_x64" // TODO: change to integ_test_linux_x64_tar ) - env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( - testType: "Integ Tests (x64, tar)", + env.ARTIFACT_URL_LINUX_X64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( + testType: "Integ Tests (linux, x64, tar)", status: integTestResults.getResult(), absoluteUrl: integTestResults.getAbsoluteUrl() ) @@ -302,11 +360,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: bwcTestResults.getId(), - stage: "bwc_test_x64" + stage: "bwc_test_x64" // TODO: change to bwc_test_linux_x64_tar ) - env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT = createTestResultsMessage( - testType: "BWC Tests (x64, tar)", + env.ARTIFACT_URL_LINUX_X64_TAR_BWC_TEST_RESULT = createTestResultsMessage( + testType: "BWC Tests (linux, 64, tar)", status: bwcTestResults.getResult(), absoluteUrl: bwcTestResults.getAbsoluteUrl() ) @@ -321,8 +379,8 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.ARTIFACT_URL_X64_TAR_INTEG_TEST_RESULT}" + - "\n${env.ARTIFACT_URL_X64_TAR_BWC_TEST_RESULT}" + "\n${env.ARTIFACT_URL_LINUX_X64_TAR_INTEG_TEST_RESULT}" + + "\n${env.ARTIFACT_URL_LINUX_X64_TAR_BWC_TEST_RESULT}" ) postCleanup() @@ -330,7 +388,13 @@ pipeline { } } } - stage('build-and-test-x64-rpm') { + stage('build-and-test-linux-x64-rpm') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { label AGENT_X64 } stages { stage('build-archive-linux-x64-rpm') { @@ -380,14 +444,14 @@ pipeline { ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_X64_RPM = artifactUrl + env.ARTIFACT_URL_LINUX_X64_RPM = artifactUrl - echo "buildManifestUrl (x64, rpm): ${buildManifestUrl}" - echo "artifactUrl (x64, rpm): ${artifactUrl}" + echo "buildManifestUrl (linux, x64, rpm): ${buildManifestUrl}" + echo "artifactUrl (linux, x64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) - echo "Trigger rpm validation for x64 rpm ${bundleManifestUrl}" + echo "Trigger rpm validation for linux x64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, @@ -397,8 +461,8 @@ pipeline { string(name: 'AGENT_LABEL', value: AGENT_X64) ] - env.RPM_VALIDATION_X64_RPM_TEST_RESULT = createTestResultsMessage( - testType: "RPM Validation (x64, rpm)", + env.RPM_VALIDATION_LINUX_X64_RPM_TEST_RESULT = createTestResultsMessage( + testType: "RPM Validation (linux, x64, rpm)", status: rpmValidationResults.getResult(), absoluteUrl: rpmValidationResults.getAbsoluteUrl() ) @@ -410,7 +474,7 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.RPM_VALIDATION_X64_RPM_TEST_RESULT}" + "\n${env.RPM_VALIDATION_LINUX_X64_RPM_TEST_RESULT}" ) postCleanup() @@ -420,7 +484,13 @@ pipeline { } } } - stage('build-and-test-arm64-tar') { + stage('build-and-test-linux-arm64-tar') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { docker { label AGENT_ARM64 @@ -434,14 +504,16 @@ pipeline { def buildManifestObj = buildAssembleUpload( componentName: "${COMPONENT_NAME}", inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'linux', + architecture: 'arm64', distribution: "tar" ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_ARM64_TAR = artifactUrl + env.ARTIFACT_URL_LINUX_ARM64_TAR = artifactUrl - echo "buildManifestUrl (arm64, tar): ${buildManifestUrl}" - echo "artifactUrl (arm64, tar): ${artifactUrl}" + echo "buildManifestUrl (linux, arm64, tar): ${buildManifestUrl}" + echo "artifactUrl (linux, arm64, tar): ${artifactUrl}" parallel([ 'integ-test': { @@ -461,11 +533,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: integTestResults.getId(), - stage: "integ_test_arm64" + stage: "integ_test_arm64" // TODO: integ_test_linux_arm64_tar ) - env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( - testType: "Integ Tests (arm64, tar)", + env.ARTIFACT_URL_LINUX_ARM64_TAR_INTEG_TEST_RESULT = createTestResultsMessage( + testType: "Integ Tests (linux, arm64, tar)", status: integTestResults.getResult(), absoluteUrl: integTestResults.getAbsoluteUrl() ) @@ -488,11 +560,11 @@ pipeline { buildInfoYaml( componentName: COMPONENT_NAME, status: bwcTestResults.getId(), - stage: "bwc_test_arm64" + stage: "bwc_test_arm64" // TODO: bwc_test_linux_arm64_tar ) - env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT = createTestResultsMessage( - testType: "BWC Tests (arm64, tar)", + env.ARTIFACT_URL_LINUX_ARM64_TAR_BWC_TEST_RESULT = createTestResultsMessage( + testType: "BWC Tests (linux, arm64, tar)", status: bwcTestResults.getResult(), absoluteUrl: bwcTestResults.getAbsoluteUrl() ) @@ -507,8 +579,8 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.ARTIFACT_URL_ARM64_TAR_INTEG_TEST_RESULT}" + - "\n${env.ARTIFACT_URL_ARM64_TAR_BWC_TEST_RESULT}" + "\n${env.ARTIFACT_URL_LINUX_ARM64_TAR_INTEG_TEST_RESULT}" + + "\n${env.ARTIFACT_URL_LINUX_ARM64_TAR_BWC_TEST_RESULT}" ) postCleanup() @@ -516,7 +588,13 @@ pipeline { } } } - stage('build-and-test-arm64-rpm') { + stage('build-and-test-linux-arm64-rpm') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('linux') + } + } agent { label AGENT_X64 } stages { stage('build-archive-linux-arm64-rpm') { @@ -566,14 +644,14 @@ pipeline { ) String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) - env.ARTIFACT_URL_ARM64_RPM = artifactUrl + env.ARTIFACT_URL_LINUX_ARM64_RPM = artifactUrl - echo "buildManifestUrl (arm64, rpm): ${buildManifestUrl}" - echo "artifactUrl (arm64, rpm): ${artifactUrl}" + echo "buildManifestUrl (linux, arm64, rpm): ${buildManifestUrl}" + echo "artifactUrl (linux, arm64, rpm): ${artifactUrl}" String bundleManifestUrl = buildManifestObj.getBundleManifestUrl(JOB_NAME, BUILD_NUMBER) - echo "Trigger rpm validation for arm64 rpm ${bundleManifestUrl}" + echo "Trigger rpm validation for linux arm64 rpm ${bundleManifestUrl}" def rpmValidationResults = build job: 'rpm-validation', propagate: false, @@ -583,8 +661,8 @@ pipeline { string(name: 'AGENT_LABEL', value: AGENT_ARM64) ] - env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT = createTestResultsMessage( - testType: "RPM Validation (arm64, rpm)", + env.RPM_VALIDATION_LINUX_ARM64_RPM_TEST_RESULT = createTestResultsMessage( + testType: "RPM Validation (linux, arm64, rpm)", status: rpmValidationResults.getResult(), absoluteUrl: rpmValidationResults.getAbsoluteUrl() ) @@ -596,7 +674,7 @@ pipeline { lib.jenkins.Messages.new(this).add( "${STAGE_NAME}", lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + - "\n${env.RPM_VALIDATION_ARM64_RPM_TEST_RESULT}" + "\n${env.RPM_VALIDATION_LINUX_ARM64_RPM_TEST_RESULT}" ) postCleanup() @@ -606,6 +684,53 @@ pipeline { } } } + stage('build-and-test-windows-x64-zip') { + when { + beforeAgent true + expression{ + params.BUILD_PLATFORM.contains('windows') + } + } + agent { + node { + label 'Jenkins-Agent-Windows2019-X64-C54xlarge-Single-Host' + } + } + tools { + jdk dockerAgent.javaVersion + } + steps { + script { + def buildManifestObj = buildAssembleUpload( + componentName: "${COMPONENT_NAME}", + inputManifest: "manifests/${INPUT_MANIFEST}", + platform: 'windows', + architecture: 'x64', + distribution: "zip" + ) + String buildManifestUrl = buildManifestObj.getUrl(JOB_NAME, BUILD_NUMBER) + String artifactUrl = buildManifestObj.getArtifactUrl(JOB_NAME, BUILD_NUMBER) + env.ARTIFACT_URL_LINUX_WINDOWS_X64_ZIP = artifactUrl + env.INDEX_FILE_PATH = buildManifestObj.getIndexFileRoot("${JOB_NAME}") + + echo "buildManifestUrl (windows, x64, zip): ${buildManifestUrl}" + echo "artifactUrl (windows, x64, zip): ${artifactUrl}" + + } + } + post { + always { + script { + lib.jenkins.Messages.new(this).add( + "${STAGE_NAME}", + lib.jenkins.Messages.new(this).get(["${STAGE_NAME}"]) + ) + + postCleanup() + } + } + } + } } } stage('update index file') { @@ -628,8 +753,13 @@ pipeline { stage('docker build') { when { beforeAgent true - expression { - params.BUILD_DOCKER != 'do_not_build_docker' + allOf{ + expression { + params.BUILD_DOCKER != 'do_not_build_docker' + } + expression{ + params.BUILD_PLATFORM.contains('linux') + } } } agent { @@ -642,15 +772,15 @@ pipeline { } steps { script { - echo "env.ARTIFACT_URL_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" - echo "env.ARTIFACT_URL_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" + echo "env.ARTIFACT_URL_LINUX_X64_TAR: ${env.ARTIFACT_URL_X64_TAR}" + echo "env.ARTIFACT_URL_LINUX_ARM64_TAR: ${env.ARTIFACT_URL_ARM64_TAR}" buildDockerImage( inputManifest: "manifests/${INPUT_MANIFEST}", buildNumber: "${BUILD_NUMBER}", buildOption: "${BUILD_DOCKER}", - artifactUrlX64: env.ARTIFACT_URL_X64_TAR, - artifactUrlArm64: env.ARTIFACT_URL_ARM64_TAR + artifactUrlX64: env.ARTIFACT_URL_LINUX_X64_TAR, + artifactUrlArm64: env.ARTIFACT_URL_LINUX_ARM64_TAR ) } } @@ -676,9 +806,9 @@ pipeline { script { if (params.PUBLISH_NOTIFICATION) { def stashed = lib.jenkins.Messages.new(this).get([ - 'build-and-test-x64-tar', + 'build-and-test-linux-x64-tar', 'assemble-archive-and-test-linux-x64-rpm', - 'build-and-test-arm64-tar', + 'build-and-test-linux-arm64-tar', 'assemble-archive-and-test-linux-arm64-rpm' ]) diff --git a/manifests/1.3.6/opensearch-1.3.6.yml b/manifests/1.3.6/opensearch-1.3.6.yml index 4c85a4158e..cd90d31ca1 100644 --- a/manifests/1.3.6/opensearch-1.3.6.yml +++ b/manifests/1.3.6/opensearch-1.3.6.yml @@ -20,24 +20,36 @@ components: checks: - gradle:publish - gradle:properties:version + platforms: + - linux + - windows - name: job-scheduler repository: https://github.com/opensearch-project/job-scheduler.git ref: '1.3' checks: - gradle:properties:version - gradle:dependencies:opensearch.version + platforms: + - linux + - windows - name: security repository: https://github.com/opensearch-project/security.git ref: '1.3' checks: - gradle:properties:version - gradle:dependencies:opensearch.version + platforms: + - linux + - windows - name: alerting repository: https://github.com/opensearch-project/alerting.git ref: '1.3' checks: - gradle:properties:version - gradle:dependencies:opensearch.version: alerting + platforms: + - linux + - windows - name: dashboards-reports repository: https://github.com/opensearch-project/dashboards-reports.git ref: '1.3' @@ -45,6 +57,9 @@ components: checks: - gradle:properties:version - gradle:dependencies:opensearch.version + platforms: + - linux + - windows - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git ref: '1.3' diff --git a/manifests/2.3.1/opensearch-2.3.1.yml b/manifests/2.3.1/opensearch-2.3.1.yml index 14d5e15b24..4592ad501d 100644 --- a/manifests/2.3.1/opensearch-2.3.1.yml +++ b/manifests/2.3.1/opensearch-2.3.1.yml @@ -19,12 +19,16 @@ components: ref: '2.3' platforms: - linux + - windows checks: - gradle:publish - gradle:properties:version - name: ml-commons repository: https://github.com/opensearch-project/ml-commons.git ref: '2.3' + platforms: + - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin @@ -33,9 +37,13 @@ components: ref: '2.3' platforms: - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version - name: security repository: https://github.com/opensearch-project/security.git ref: '2.3' + platforms: + - linux + - windows diff --git a/manifests/2.4.0/opensearch-2.4.0.yml b/manifests/2.4.0/opensearch-2.4.0.yml index 4d8cc37778..32381797e1 100644 --- a/manifests/2.4.0/opensearch-2.4.0.yml +++ b/manifests/2.4.0/opensearch-2.4.0.yml @@ -19,6 +19,7 @@ components: ref: 2.x platforms: - linux + - windows checks: - gradle:publish - gradle:properties:version @@ -27,12 +28,16 @@ components: ref: 2.x platforms: - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin - name: security repository: https://github.com/opensearch-project/security.git ref: 2.x + platforms: + - linux + - windows - name: performance-analyzer repository: https://github.com/opensearch-project/performance-analyzer.git ref: 2.x @@ -46,6 +51,7 @@ components: ref: '2.x' platforms: - linux + - windows working_directory: notifications checks: - gradle:properties:version @@ -55,6 +61,7 @@ components: ref: '2.x' platforms: - linux + - windows working_directory: notifications checks: - gradle:properties:version @@ -64,6 +71,7 @@ components: ref: '2.x' platforms: - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version @@ -72,6 +80,7 @@ components: ref: '2.x' platforms: - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version: alerting diff --git a/manifests/3.0.0/opensearch-3.0.0.yml b/manifests/3.0.0/opensearch-3.0.0.yml index a7f0d3adbe..92689310cc 100644 --- a/manifests/3.0.0/opensearch-3.0.0.yml +++ b/manifests/3.0.0/opensearch-3.0.0.yml @@ -17,18 +17,27 @@ components: - name: common-utils repository: https://github.com/opensearch-project/common-utils.git ref: main + platforms: + - linux + - windows checks: - gradle:publish - gradle:properties:version - name: ml-commons repository: https://github.com/opensearch-project/ml-commons.git ref: main + platforms: + - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version: opensearch-ml-plugin - name: job-scheduler repository: https://github.com/opensearch-project/job-scheduler.git ref: main + platforms: + - linux + - windows checks: - gradle:properties:version - gradle:dependencies:opensearch.version