Skip to content

Commit

Permalink
[Build] Perform native binaries build as part of the Jenkins pipeline
Browse files Browse the repository at this point in the history 
This changes the Jenkins pipeline of the equinox repository to build the
native binaries automatically if the native sources have been changed
since the last natives-build. It does the same as the existing Jenkins
free-style jobs managed only in the Jenkins-UI and triggered manually
and consequently makes them obsolete.
If the build is for the 'master' or a maintanance-branch the built
native binaries are pushed to the equinox.binaries repository, choosing
the correct target branch automatically.

What is not supported any more:
- partial builds only for one specific platform
- versions are always incremented

Fixes eclipse-equinox#575
  • Loading branch information
@HannesWell
HannesWell committed May 3, 2024
1 parent bc6a759 commit aebd43e
Show file tree
Hide file tree
Showing 2 changed files with 258 additions and 4 deletions.
253 changes: 253 additions & 0 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,58 @@
*
* Contributors:
* Mickael Istria (Red Hat Inc.) - initial API and implementation
* Hannes Wellmann - Build Equinox native launchers and executables on demand as part of master- and verification-builds
*******************************************************************************/

def nativeBuildAgent(String platform, Closure body) {
def final nativeBuildStageName = 'Build Launcher native binaries'
if (platform == 'gtk.linux.x86_64') {
return podTemplate(yaml: '''
apiVersion: v1
kind: Pod
spec:
containers:
- name: "launcherbuild"
image: "eclipse/platformreleng-centos-swt-build:8"
imagePullPolicy: "Always"
command:
- cat
tty: true
volumeMounts:
- name: tools
mountPath: /opt/tools
volumes:
- name: tools
persistentVolumeClaim:
claimName: tools-claim-jiro-releng
- name: volume-known-hosts
configMap:
name: known-hosts
''') { node(POD_LABEL) { stage(nativeBuildStageName) { container('launcherbuild') { body() } } } }
} else {
if (platform == 'cocoa.macosx.x86_64') {
platform = 'cocoa.macosx.aarch64'
}
return node('swt.natives-' + platform) { stage(nativeBuildStageName) { body() } } //TODO: generalize labels
}
}

/** Returns the download URL of the JDK against whoose C headers (in the 'include/' folder) and native libaries the natives are compiled.*/
def getNativeJdkUrl(String os, String arch) { // To update the used JDK version update the URL template below
return "https://download.eclipse.org/justj/jres/17/downloads/20230428_1804/org.eclipse.justj.openjdk.hotspot.jre.minimal.stripped-17.0.7-${os}-${arch}.tar.gz"
}

//TODO: adjust tag names to be based on the binaries version instead of a time-stamp?
def getLatestGitTag() {
return sh(script: 'git describe --abbrev=0 --tags --match v[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9]', returnStdout: true).strip()
}

def isOnMainIshBranch() {
return env.BRANCH_NAME == ('master') || env.BRANCH_NAME ==~ 'R[0-9]+_[0-9]+_maintenance'
}

boolean BUILD_NATIVES = false

pipeline {
options {
skipDefaultCheckout() // Specialiced checkout is performed below
Expand All @@ -34,11 +80,30 @@ pipeline {
environment {
EQUINOX_BINARIES_LOC = "$WORKSPACE/equinox.binaries"
}
parameters {
booleanParam(name: 'forceNativeBuilds', defaultValue: false, description: 'Forces to run the native builds of Equinox\' binaries. Will push the built binaries to the master branch, unless \'skipCommit\' is set. Useful in debugging.')
booleanParam(name: 'skipCommit', defaultValue: false, description: 'Stops committing to equinox and equinox binaries repo at the end. Useful in debugging.')
}
stages {
stage('Checkout SCM') {
steps{
dir('equinox') {
checkout scm
script {
def authorMail = sh(script: 'git log -1 --pretty=format:"%ce" HEAD', returnStdout: true)
echo 'HEAD commit author: ' + authorMail
def buildBotMail = 'eclipse-releng-bot@eclipse.org'
if (buildBotMail.equals(authorMail) && !params.forceNativeBuilds) {
// Prevent endless build-loops due to self triggering because of a previous automated native-build and the associated updates.
currentBuild.result = 'ABORTED'
error('Abort build only triggered by automated natives update.')
}
sh """
git config --global user.email '${buildBotMail}'
git config --global user.name 'Eclipse Releng Bot'
git fetch --all --tags --quiet
"""
}
}
dir('equinox.binaries') {
checkout([$class: 'GitSCM',
Expand All @@ -50,6 +115,161 @@ pipeline {
}
}
}
stage('Check if native launchers and executables changed') {
steps{
dir('equinox') {
script {
def nativeBuildTag = getLatestGitTag()
echo "Last tag: ${nativeBuildTag}"
def diff = sh(script: "git diff HEAD ${nativeBuildTag} -- 'features/org.eclipse.equinox.executable.feature/library'", returnStdout: true)
boolean nativesChanged = !diff.strip().isEmpty()
echo "Natives changed since ${nativeBuildTag}: ${nativesChanged}"
if (nativesChanged || params.forceNativeBuilds) {
BUILD_NATIVES = true
dir('features/org.eclipse.equinox.executable.feature/library') {
sh '''
old_ver=$(grep "min_ver=" make_version.mak |cut -d= -f2)
new_ver=$(expr $old_ver + 1)
sed -i -e "s/$old_ver/$new_ver/" make_version.mak
'''
stash(name:"equinox.binaries.sources.cocoa", includes: '*,cocoa/')
stash(name:"equinox.binaries.sources.gtk", includes: '*,gtk/')
stash(name:"equinox.binaries.sources.win32", includes: '*,win32/')
}
}
}
}
}
}
stage('Build launcher and executable binaries, if needed') {
when {
expression { BUILD_NATIVES }
}
matrix {
axes {
axis {
name 'PLATFORM'
values 'cocoa.macosx.aarch64' , 'cocoa.macosx.x86_64', 'gtk.linux.aarch64', 'gtk.linux.ppc64le', 'gtk.linux.x86_64', 'win32.win32.x86_64'
}
}
stages {
stage('Build Equinox natives') {
options {
timeout(time: 120, unit: 'MINUTES') // Some build agents are rare and it might take awhile until they are available.
}
steps {
script {
def (ws, os, arch) = env.PLATFORM.split('\\.')
dir("jdk-download-${os}.${arch}") {
// Fetch the JDK, which provides the C header-files and shared native libaries, against which the natives are built.
sh "curl ${getNativeJdkUrl(os, arch)} | tar -xzf - include/ lib/"
stash(name:"jdk.resources.${os}.${arch}", includes: "include/,lib/")
deleteDir()
}
nativeBuildAgent("${PLATFORM}") {
cleanWs() // Workspace not cleaned by default
echo "OS: ${os}, ARCH: ${arch}"
unstash("equinox.binaries.sources.${ws}")
dir('jdk.resources') {
unstash("jdk.resources.${os}.${arch}")
}
withEnv(["JAVA_HOME=${WORKSPACE}/jdk.resources", "EXE_OUTPUT_DIR=${WORKSPACE}/libs", "LIB_OUTPUT_DIR=${WORKSPACE}/libs"]) {
dir(ws) {
if (isUnix()) {
sh "sh build.sh -ws ${ws} -os ${os} -arch ${arch} install"
} else {
bat "cmd /c build.bat -ws ${ws} -os ${os} -arch ${arch} install"
}
}
}
dir('libs') {
stash("equinox.binaries.${PLATFORM}")
}
}
}
}
}
stage('Collect and sign binaries') {
steps {
dir("libs/${PLATFORM}") {
unstash("equinox.binaries.${PLATFORM}")
script {
def (ws, os, arch) = env.PLATFORM.split('\\.')
withEnv(["ws=${ws}", "os=${os}", "arch=${arch}", "isMainIshBranch=${isOnMainIshBranch()}"]) {
sh '''
fnSignFile()
{
filename=$1
signerUrl=$2
extraArguments=$3
if [[ ${isMainIshBranch} == true ]]; then
mv ${filename} unsigned-${filename}
curl --fail --form "file=@unsigned-${filename}" --output "${filename}" ${extraArguments} "${signerUrl}"
rm unsigned-${filename}
fi
}
binPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.executable/bin/${ws}/${os}/${arch}
libPath=${WORKSPACE}/equinox.binaries/org.eclipse.equinox.launcher.${ws}.${os}.${arch}
if [[ ${PLATFORM} == cocoa.macosx.* ]]; then
libFileExt='so'
signerUrl='https://cbi.eclipse.org/macos/codesign/sign'
binPath=${binPath}/Eclipse.app/Contents/MacOS
# Sign executable for Mac
curl --output 'sdk.entitlement' https://download.eclipse.org/eclipse/relengScripts/entitlement/sdk.entitlement
fnSignFile eclipse "${signerUrl}" '-F entitlements=@sdk.entitlement'
elif [[ ${PLATFORM} == gtk.linux.* ]]; then
libFileExt='so'
elif [[ ${PLATFORM} == win32.win32.* ]]; then
exeFileExt='*.exe'
libFileExt='dll'
signerUrl='https://cbi.eclipse.org/authenticode/sign'
fi
if [[ -n "${signerUrl}" ]]; then
echo "Sign ${PLATFORM} libraries"
for file in *.${libFileExt}; do
fnSignFile "${file}" "${signerUrl}"
done
fi
mkdir -p ${binPath}
mkdir -p ${libPath}
echo 'Clean existing binaries'
rm -rf ${binPath}/eclipse${exeFileExt}
rm -rf ${libPath}/eclipse_*.${libFileExt}
echo "Copy new binaries"
mv eclipse${exeFileExt} ${binPath}
mv eclipse_*.${libFileExt} ${libPath}
'''
}
}
}
}
}
}
}
}
stage('Commit native binaries, if built') {
when {
expression { BUILD_NATIVES }
}
steps {
dir('equinox.binaries') {
withAnt(installation: 'apache-ant-latest', jdk: 'openjdk-jdk11-latest') { // nashorn javascript-engine required in ant-scripts
sh '''
cd ${WORKSPACE}/equinox/releng/org.eclipse.equinox.launcher.releng/
ant commitBinaries -DlocalGit=${WORKSPACE}
'''
}
}
}
}
stage('Build') {
steps {
dir('equinox') {
Expand All @@ -69,5 +289,38 @@ pipeline {
}
}
}
stage('Push Equinox-native binaries, if built') {
when {
expression { BUILD_NATIVES }
}
steps {
sshagent(['github-bot-ssh']) {
sh """
# Check for the master-branch as late as possible to have as much of the same behaviour as possible
if [[ ${isOnMainIshBranch()} == true ]]; then
if [[ ${params.skipCommit} != true ]]; then
# Don't rebase and just fail in case another commit has been pushed to the master/maintanance branch in the meantime
pushd equinox
git push origin HEAD:refs/heads/${BRANCH_NAME}
git push origin refs/tags/${getLatestGitTag()}
popd
pushd equinox.binaries
git push origin HEAD:refs/heads/${BRANCH_NAME}
git push origin refs/tags/${getLatestGitTag()}
popd
exit 0
else
echo Committing is skipped
fi
else
echo Skip pushing changes of native-binaries for branch '${BRANCH_NAME}'
fi
"""
}
}
}
}
}
9 changes: 5 additions & 4 deletions releng/org.eclipse.equinox.launcher.releng/build.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,12 +240,13 @@
<property name="tag" value="v${DSTAMP}-${TSTAMP}"/>
<property name="commitMsg" value="" />

<git dir="${localGit}/rt.equinox.binaries" command="commit" arguments="-a -m &quot;Recompiled binaries ${commitMsg}&quot;" />
<git dir="${localGit}/rt.equinox.binaries" command="tag" arguments="${tag}" />
<git dir="${localGit}/equinox.binaries" command="commit" arguments="-a -m &quot;Recompiled binaries ${commitMsg}&quot;" />
<git dir="${localGit}/equinox.binaries" command="tag" arguments="${tag}" />

<replaceregexp match="binaryTag=(.*)" replace="binaryTag=${tag}" >
<fileset dir="${localGit}/rt.equinox.framework" includes="**/build.properties" />
<fileset dir="${localGit}/equinox" includes="**/build.properties" />
</replaceregexp>
<git dir="${localGit}/rt.equinox.framework" command="commit" arguments="-a -m &quot;Binaries ${tag} ${commitMsg}&quot;" />
<git dir="${localGit}/equinox" command="commit" arguments="-a -m &quot;Binaries ${tag} ${commitMsg}&quot;" />
<git dir="${localGit}/equinox" command="tag" arguments="${tag}" />
</target>
</project>

0 comments on commit aebd43e

Please sign in to comment.