From 040f4bc1d7e8cabb6b948640b5a03bc899bce77d Mon Sep 17 00:00:00 2001 From: Hannes Wellmann Date: Fri, 29 Mar 2024 15:42:33 +0100 Subject: [PATCH] [Build] Perform native binaries build as part of the Jenkins pipeline This changes the Jenkins pipeline of the equinox repository to build the native binaries automatically if the native sources have been changed since the last natives-build. It does the same as the existing Jenkins free-style jobs managed only in the Jenkins-UI and triggered manually and consequently makes them obsolete. If the build is for the 'master' or a maintanance-branch the built native binaries are pushed to the equinox.binaries repository, choosing the correct target branch automatically. What is not supported any more: - partial builds only for one specific platform - versions are always incremented Fixes https://github.com/eclipse-equinox/equinox/issues/575 --- Jenkinsfile | 252 ++++++++++++++++++ .../build.xml | 9 +- 2 files changed, 257 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 395a8d8c6d1..26ead393d15 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,12 +10,58 @@ * * Contributors: * Mickael Istria (Red Hat Inc.) - initial API and implementation + * Hannes Wellmann - Build Equinox native launchers and executables on demand as part of master- and verification-builds *******************************************************************************/ +def nativeBuildAgent(String platform, Closure body) { + def final nativeBuildStageName = 'Build Launcher native binaries' + if (platform == 'gtk.linux.x86_64') { + return podTemplate(yaml: ''' +apiVersion: v1 +kind: Pod +spec: + containers: + - name: "launcherbuild" + image: "eclipse/platformreleng-centos-swt-build:8" + imagePullPolicy: "Always" + command: + - cat + tty: true + volumeMounts: + - name: tools + mountPath: /opt/tools + volumes: + - name: tools + persistentVolumeClaim: + claimName: tools-claim-jiro-releng + - name: volume-known-hosts + configMap: + name: known-hosts +''') { node(POD_LABEL) { stage(nativeBuildStageName) { container('launcherbuild') { body() } } } } + } else { + if (platform == 'cocoa.macosx.x86_64') { + platform = 'cocoa.macosx.aarch64' + } + return node('swt.natives-' + platform) { stage(nativeBuildStageName) { body() } } //TODO: generalize labels + } +} + +/** Returns the download URL of the JDK against whoose C headers (in the 'include/' folder) and native libaries the natives are compiled.*/ +def getNativeJdkUrl(String os, String arch) { // To update the used JDK version update the URL template below + return "https://download.eclipse.org/justj/jres/17/downloads/20230428_1804/org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped-17.0.7-${os}-${arch}.tar.gz" +} + +//TODO: adjust tag names to be based on the binaries version instead of a time-stamp? +def getLatestGitTag() { + return sh(script: 'git describe --abbrev=0 --tags --match v[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]', returnStdout: true).strip() +} + def isOnMainIshBranch() { return env.BRANCH_NAME == ('master') || env.BRANCH_NAME ==~ 'R[0-9]+_[0-9]+_maintenance' } +boolean BUILD_NATIVES = false + pipeline { options { skipDefaultCheckout() // Specialiced checkout is performed below @@ -34,11 +80,29 @@ pipeline { environment { EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries" } + parameters { + booleanParam(name: 'forceNativeBuilds', defaultValue: false, description: 'Forces to run the native builds of Equinox\' binaries. Will push the built binaries to the master branch, unless \'skipCommit\' is set. Useful in debugging.') + booleanParam(name: 'skipCommit', defaultValue: false, description: 'Stops committing to equinox and equinox binaries repo at the end. Useful in debugging.') + } stages { stage('Checkout SCM') { steps{ dir('equinox') { checkout scm + script { + def authorMail = sh(script: 'git log -1 --pretty=format:"%ce" HEAD', returnStdout: true) + echo 'HEAD commit author: ' + authorMail + def buildBotMail = 'eclipse-releng-bot@eclipse.org' + if (buildBotMail.equals(authorMail) && !params.forceNativeBuilds) { + // Prevent endless build-loops due to self triggering because of a previous automated native-build and the associated updates. + currentBuild.result = 'ABORTED' + error('Abort build only triggered by automated natives update.') + } + sh """ + git config --global user.email '${buildBotMail}' + git config --global user.name 'Eclipse Releng Bot' + """ + } } dir('equinox.binaries') { checkout([$class: 'GitSCM', @@ -50,6 +114,161 @@ pipeline { } } } + stage('Check if native launchers and executables changed') { + steps{ + dir('equinox') { + script { + def nativeBuildTag = getLatestGitTag() + echo "Last tag: ${nativeBuildTag}" + def diff = sh(script: "git diff HEAD ${nativeBuildTag} -- 'features/org.eclipse.equinox.executable.feature/library'", returnStdout: true) + boolean nativesChanged = !diff.strip().isEmpty() + echo "Natives changed since ${nativeBuildTag}: ${nativesChanged}" + if (nativesChanged || params.forceNativeBuilds) { + BUILD_NATIVES = true + dir('features/org.eclipse.equinox.executable.feature/library') { + sh ''' + old_ver=$(grep "min_ver=" make_version.mak |cut -d= -f2) + new_ver=$(expr $old_ver + 1) + sed -i -e "s/$old_ver/$new_ver/" make_version.mak + ''' + stash(name:"equinox.binaries.sources.cocoa", includes: '*,cocoa/') + stash(name:"equinox.binaries.sources.gtk", includes: '*,gtk/') + stash(name:"equinox.binaries.sources.win32", includes: '*,win32/') + } + } + } + } + } + } + stage('Build launcher and executable binaries, if needed') { + when { + expression { BUILD_NATIVES } + } + matrix { + axes { + axis { + name 'PLATFORM' + values 'cocoa.macosx.aarch64' , 'cocoa.macosx.x86_64', 'gtk.linux.aarch64', 'gtk.linux.ppc64le', 'gtk.linux.x86_64', 'win32.win32.x86_64' + } + } + stages { + stage('Build Equinox natives') { + options { + timeout(time: 120, unit: 'MINUTES') // Some build agents are rare and it might take awhile until they are available. + } + steps { + script { + def (ws, os, arch) = env.PLATFORM.split('\\.') + dir("jdk-download-${os}.${arch}") { + // Fetch the JDK, which provides the C header-files and shared native libaries, against which the natives are build. + sh "curl ${getNativeJdkUrl(os, arch)} | tar -xzf - include/ lib/" + stash(name:"jdk.resources.${os}.${arch}", includes: "include/,lib/") + deleteDir() + } + nativeBuildAgent("${PLATFORM}") { + cleanWs() // Workspace not cleaned by default + echo "OS: ${os}, ARCH: ${arch}" + unstash("equinox.binaries.sources.${ws}") + dir('jdk.resources') { + unstash("jdk.resources.${os}.${arch}") + } + withEnv(["JAVA_HOME=${WORKSPACE}/jdk.resources", "EXE_OUTPUT_DIR=${WORKSPACE}/libs", "LIB_OUTPUT_DIR=${WORKSPACE}/libs"]) { + dir(ws) { + if (isUnix()) { + sh "sh build.sh -ws ${ws} -os ${os} -arch ${arch} install" + } else { + bat "cmd /c build.bat -ws ${ws} -os ${os} -arch ${arch} install" + } + } + } + dir('libs') { + stash("equinox.binaries.${PLATFORM}") + } + } + } + } + } + stage('Collect and sign binaries') { + steps { + dir("libs/${PLATFORM}") { + unstash("equinox.binaries.${PLATFORM}") + script { + def (ws, os, arch) = env.PLATFORM.split('\\.') + withEnv(["ws=${ws}", "os=${os}", "arch=${arch}", "isMainIshBranch=${isOnMainIshBranch()}"]) { + sh ''' + fn-sign-file () + { + filename=$1 + signerUrl=$2 + extraArguments=$3 + if [[ ${isMainIshBranch} == true ]]; then + mv ${filename} unsigned-${filename} + curl --fail --form "file=@unsigned-${filename}" --output "${filename}" ${extraArguments} "${signerUrl}" + rm unsigned-${filename} + fi + } + + binPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.executable/bin/${ws}/${os}/${arch} + libPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.launcher.${ws}.${os}.${arch} + + if [[ ${PLATFORM} == cocoa.macosx.* ]]; then + libFileExt='so' + signerUrl='https://cbi.eclipse.org/macos/codesign/sign' + binPath = ${binPath}/Eclipse.app/Contents/MacOS + # Sign executable for Mac + wget https://download.eclipse.org/eclipse/relengScripts/entitlement/sdk.entitlement + fn-sign-file eclipse "${signerUrl}" '-F entitlements=@sdk.entitlement' + + elif [[ ${PLATFORM} == gtk.linux.* ]]; then + libFileExt='so' + + elif [[ ${PLATFORM} == win32.win32.* ]]; then + exeFileExt='*.exe' + libFileExt='dll' + signerUrl='https://cbi.eclipse.org/authenticode/sign' + fi + + if [[ -n "${signerUrl}" ]]; then + echo "Sign ${PLATFORM} libraries" + for file in *.${libFileExt}; do + fn-sign-file "${file}" "${signerUrl}" + done + fi + + mkdir -p ${binPath} + mkdir -p ${libPath} + + echo 'Clean existing binaries' + rm -rf ${binPath}/eclipse${exeFileExt} + rm -rf ${libPath}/eclipse_*.${libFileExt} + + echo "Copy new binaries" + mv eclipse${exeFileExt} ${binPath} + mv eclipse_*.${libFileExt} ${libPath} + ''' + } + } + } + } + } + } + } + } + stage('Commit native binaries, if build') { + when { + expression { BUILD_NATIVES } + } + steps { + dir('equinox.binaries') { + withAnt(installation: 'apache-ant-latest', jdk: 'openjdk-jdk11-latest') { // nashorn javascript-engine required in ant-scripts + sh ''' + cd ${WORKSPACE}/equinox/releng/org.eclipse.equinox.launcher.releng/ + ant commitBinaries -DlocalGit=${WORKSPACE} + ''' + } + } + } + } stage('Build') { steps { dir('equinox') { @@ -69,5 +288,38 @@ pipeline { } } } + stage('Push Equinox-native binaries, if build') { + when { + expression { BUILD_NATIVES } + } + steps { + sshagent(['github-bot-ssh']) { + sh """ + # Check for the master-branch as late as possible to have as much of the same behaviour as possible + if [[ ${isOnMainIshBranch()} == true ]]; then + if [[ ${params.skipCommit} != true ]]; then + + # Don't rebase and just fail in case another commit has been pushed to the master/maintanance branch in the meantime + pushd equinox + git push origin HEAD:refs/heads/${BRANCH_NAME} + git push origin refs/tags/${getLatestGitTag()} + popd + + pushd equinox.binaries + git push origin HEAD:refs/heads/${BRANCH_NAME} + git push origin refs/tags/${getLatestGitTag()} + popd + + exit 0 + else + echo Committing is skipped + fi + else + echo Skip pushing changes of native-binaries for branch '${BRANCH_NAME}' + fi + """ + } + } + } } } diff --git a/releng/org.eclipse.equinox.launcher.releng/build.xml b/releng/org.eclipse.equinox.launcher.releng/build.xml index 071ebd1afee..534956c5aba 100644 --- a/releng/org.eclipse.equinox.launcher.releng/build.xml +++ b/releng/org.eclipse.equinox.launcher.releng/build.xml @@ -240,12 +240,13 @@ - - + + - + - + +