diff --git a/.github/release-drafter.yml b/.github/release-drafter.yml index b47e0a168..f310ee890 100644 --- a/.github/release-drafter.yml +++ b/.github/release-drafter.yml @@ -1,8 +1,3 @@ _extends: .github -tag-template: bom-$NEXT_PATCH_VERSION -version-template: $MAJOR.$MINOR.$PATCH -branches: - - master - - 2.164.x - - 2.150.x - - 2.138.x +tag-template: bom-$NEXT_MAJOR_VERSION +version-template: $MAJOR diff --git a/Jenkinsfile b/Jenkinsfile index 3f73566ba..20f526156 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -17,35 +17,49 @@ def mavenEnv(body) { } def plugins +def lines stage('prep') { mavenEnv { checkout scm - def tmp = pwd tmp: true - withEnv(["SAMPLE_PLUGIN_OPTS=-Dmaven.repo.local=$tmp/m2repo -Dset.changelist -Dexpression=changelist -Doutput=$tmp/changelist help:evaluate"]) { + withEnv(['SAMPLE_PLUGIN_OPTS=-Dset.changelist']) { sh 'bash prep.sh' } - dir('sample-plugin/target') { + dir('target') { plugins = readFile('plugins.txt').split(' ') - stash name: 'pct', includes: 'megawar.war,pct.jar' - } - stash name: 'ci', includes: 'pct.sh' - def changelist = readFile("$tmp/changelist") - dir("$tmp/m2repo") { - archiveArtifacts artifacts: "**/*$changelist/*$changelist*", excludes: '**/sample/' + lines = readFile('lines.txt').split(' ') + lines = [lines[0], lines[-1]] // run PCT only on newest and oldest lines, to save resources + stash name: 'pct', includes: 'pct.jar' + lines.each {stash name: "megawar-$it", includes: "megawar-${it}.war"} } + stash name: 'pct.sh', includes: 'pct.sh' + infra.prepareToPublishIncrementals() } } +// TODO would much rather parallelize *all* PCT tests, but (INFRA-2283) ci.jenkins.io just falls over when we try. +// Running in parallel by plugin but serially by line works, albeit slowly, since workflow-cps is a bottleneck. +// So we try to manually constrain parallelism. +def semaphore = 30 // 50× parallelism usually works; 84× seems to fail reliably. branches = [failFast: true] -plugins.each { plugin -> - branches["pct-$plugin"] = { - mavenEnv { - deleteDir() - unstash 'ci' - unstash 'pct' - withEnv(["PLUGINS=$plugin"]) { - sh 'bash pct.sh' +lines.each {line -> + plugins.each { plugin -> + branches["pct-$plugin-$line"] = { + // TODO JENKINS-29037 would be useful here to wait with a longer period + waitUntil {if (semaphore > 0) {semaphore--; true} else {false}} // see JENKINS-27127 + assert semaphore >= 0 + try { + mavenEnv { + deleteDir() + unstash 'pct.sh' + unstash 'pct' + unstash "megawar-$line" + withEnv(["PLUGINS=$plugin", "LINE=$line"]) { + sh 'mv megawar-$LINE.war megawar.war && bash pct.sh' + } + } + } finally { + semaphore++ } } } diff --git a/README.md b/README.md index 316fb8305..aa71adc34 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,8 @@ just import the [latest BOM](https://github.com/jenkinsci/bom/releases) from tha io.jenkins.tools.bom - bom - 2.138.1 + bom-2.138.x + 3 import pom @@ -29,11 +29,6 @@ just import the [latest BOM](https://github.com/jenkinsci/bom/releases) from tha ``` -(The patch component of the BOM version, `1` in this example, -is unrelated to the patch component of the Jenkins LTS version, `4` in this example. -Generally you should select the latest of each patch component independently. -The major and minor components, in this example `2` and `138` respectively, must match.) - Now you can declare dependencies on many plugins without needing to specify a version: ```xml @@ -131,51 +126,31 @@ It is unusual but possible for cross-component incompatibilities to only be visi ## LTS lines -The `master` branch should track the current LTS line. -A few historical lines are also tracked by branches, -for use from plugins which are not yet ready to depend on the latest. -Each line is released independently with `maven-release-plugin`. -When a new LTS line is released (`jenkins-2.xxx.1`), -a new BOM branch should be cut from the current `master`, -and `master` made to track the new line. +A separate BOM artifact if available for the current LTS line and a few historical lines. +When a new LTS line is released (`jenkins-2.nnn.1`), +the main definition should be moved into its BOM, +and the the previous BOM made to inherit from it. +Older BOMs should only specify plugin version overrides compared to the next-newer BOM. +`sample-plugin` also needs a POM profile for each supported line. -The CI build (or just `mvn test`) will fail if some managed plugins are too new for the LTS line. +The CI build (or just `mvn test -P2.nnn.x`) will fail if some managed plugins are too new for the LTS line. [This script](https://gist.github.com/jglick/0a85759ea65f60e107ac5a85a5032cae) is a handy way to find the most recently released plugin version compatible with a given line, according to the `jenkins-infra/update-center2` (which currently maintains releases for the past five lines). -General changes (such as to CI infrastructure), and most dependency updates, should be done in `master` first. -Commits from `master` should be merged into the next older LTS branch, -and from there into the branch one older, and so on. -This ensures that CI-related changes propagate to all branches without manual copy-and-paste. -Merge conflicts should be resolved in favor of the `HEAD`, -so that the branches differ from `master` only in POMs (and perhaps in sample plugin code). - -To be safe, rather than directly pushing merges, prepare them in a PR branch: - -```sh -git checkout -b 2.164.x-merge 2.164.x -git merge master -git push fork -# file a PR from youracct:2.164.x-merge → jenkinsci:2.164.x -git checkout -b 2.150.x-merge 2.150.x -git merge 2.164.x-merge -git push fork -# etc. -``` - -and only merge the PR if CI passes. - ## Releasing `release:prepare` only runs basic tests about plugin versions, not the full PCT. -Therefore be sure to check [commit status for the selected branch](https://github.com/jenkinsci/bom/commits/master) +Therefore be sure to check [commit status](https://github.com/jenkinsci/bom/commits/master) to ensure that CI builds have passed before cutting a release. Due to a misconfiguration in Incrementals tooling (JENKINS-58641), -currently after every release you must manually edit `sample-plugin/pom.xml` -and reset `version` to `${revision}${changelist}` -and set `revision` to that of the top-level `pom.xml`. +currently after every release you must manually run + +```bash +mvn -f sample-plugin incrementals:reincrementalify +``` + Commit and push the result to fix the branch build. ## Incrementals diff --git a/bom-2.138.x/pom.xml b/bom-2.138.x/pom.xml new file mode 100644 index 000000000..46a2b48e2 --- /dev/null +++ b/bom-2.138.x/pom.xml @@ -0,0 +1,57 @@ + + + 4.0.0 + + io.jenkins.tools.bom + parent + ${revision}${changelist} + + bom-2.138.x + pom + + + + ${project.groupId} + bom-2.150.x + ${project.version} + import + pom + + + org.jenkins-ci.plugins.workflow + workflow-basic-steps + 2.16.1 + + + org.jenkins-ci.plugins.workflow + workflow-durable-task-step + 2.28 + + + org.jenkins-ci.plugins + ansicolor + 0.5.3 + + + org.jenkins-ci.plugins + credentials-binding + 1.18 + + + org.jenkins-ci.plugins + mailer + 1.23 + + + org.jenkins-ci.plugins + ssh-slaves + 1.29.4 + + + org.jenkins-ci.plugins + timestamper + 1.9 + + + + diff --git a/bom-2.150.x/pom.xml b/bom-2.150.x/pom.xml new file mode 100644 index 000000000..976f892b9 --- /dev/null +++ b/bom-2.150.x/pom.xml @@ -0,0 +1,22 @@ + + + 4.0.0 + + io.jenkins.tools.bom + parent + ${revision}${changelist} + + bom-2.150.x + pom + + + + ${project.groupId} + bom-2.164.x + ${project.version} + import + pom + + + + diff --git a/bom-2.164.x/pom.xml b/bom-2.164.x/pom.xml new file mode 100644 index 000000000..d37751557 --- /dev/null +++ b/bom-2.164.x/pom.xml @@ -0,0 +1,27 @@ + + + 4.0.0 + + io.jenkins.tools.bom + parent + ${revision}${changelist} + + bom-2.164.x + pom + + + + ${project.groupId} + bom-2.176.x + ${project.version} + import + pom + + + org.jenkins-ci.plugins.workflow + workflow-durable-task-step + 2.31 + + + + diff --git a/bom/pom.xml b/bom-2.176.x/pom.xml similarity index 95% rename from bom/pom.xml rename to bom-2.176.x/pom.xml index bdf7093fc..6df743555 100644 --- a/bom/pom.xml +++ b/bom-2.176.x/pom.xml @@ -6,7 +6,7 @@ parent ${revision}${changelist} - bom + bom-2.176.x pom 1.30 @@ -281,20 +281,4 @@ - - - - org.codehaus.mojo - flatten-maven-plugin - - - flatten - - bom - - - - - - diff --git a/local-test.sh b/local-test.sh index 337597803..cef18a0f1 100644 --- a/local-test.sh +++ b/local-test.sh @@ -2,14 +2,22 @@ set -euxo pipefail cd $(dirname $0) -# expects: $PLUGINS, optionally $TEST +# expects: $PLUGINS, optionally $TEST, $LINE + +LATEST_LINE=$(ls -1d bom-*.x | sort -rn | head -1 | sed s/bom-//g) +: "${LINE:=$LATEST_LINE}" export SAMPLE_PLUGIN_OPTS=-Dtest=InjectedTest -bash prep.sh +if [ $LINE \!= $LATEST_LINE ] +then + export SAMPLE_PLUGIN_OPTS="$SAMPLE_PLUGIN_OPTS -P$LINE" +fi +LINEZ=$LINE bash prep.sh rm -rf target/local-test mkdir target/local-test -cp -v sample-plugin/target/{megawar.war,pct.jar} pct.sh target/local-test +cp -v target/pct.jar pct.sh target/local-test +cp -v target/megawar-$LINE.war target/local-test/megawar.war if [ -v TEST ] then @@ -29,11 +37,12 @@ then -e MAVEN_OPTS=-Duser.home=/var/maven \ -e MAVEN_CONFIG=/var/maven/.m2 \ -e PLUGINS=$PLUGINS \ + -e LINE=$LINE \ -e EXTRA_MAVEN_PROPERTIES=$EXTRA_MAVEN_PROPERTIES \ --entrypoint bash \ jenkins/jnlp-agent-maven \ -c "trap 'chown -R $(id -u):$(id -g) /pct /var/maven/.m2/repository' EXIT; bash /pct/pct.sh" else export EXTRA_MAVEN_PROPERTIES - bash target/local-test/pct.sh + LINE=$LINE bash target/local-test/pct.sh fi diff --git a/pct.sh b/pct.sh index 1ba8b8a89..2bc8f7550 100644 --- a/pct.sh +++ b/pct.sh @@ -2,7 +2,7 @@ set -euxo pipefail cd $(dirname $0) -# expects: megawar.war, pct.war, $PLUGINS +# expects: megawar.war, pct.war, $PLUGINS, $LINE rm -rf pct-work pct-report.xml @@ -55,5 +55,13 @@ rm -fv pct-work/durable-task/target/surefire-reports/TEST-org.jenkinsci.plugins. rm -fv pct-work/git-client/target/surefire-reports/TEST-hudson.plugins.git.GitExceptionTest.xml # TODO fails for one reason in (non-PCT) official sources, run locally; and for another reason in PCT in Docker; passes in official sources in Docker, or locally in PCT rm -fv pct-work/git-client/target/surefire-reports/TEST-org.jenkinsci.plugins.gitclient.FilePermissionsTest.xml +if [ $LINE = 2.138.x ] +then + # TODO pending backport of https://github.com/jenkinsci/workflow-durable-task-step-plugin/pull/95 from 2.29 to 2.28.x, workflow-support 3.x breaks ExecutorStepTest.serialForm + rm -fv pct-work/workflow-durable-task-step/target/surefire-reports/TEST-org.jenkinsci.plugins.workflow.support.steps.ExecutorStepTest.xml + # TODO pending ssh-slaves 1.30.0 with https://github.com/jenkinsci/ssh-slaves-plugin/pull/114 (plus https://github.com/jenkinsci/durable-task-plugin/pull/100): + rm -fv pct-work/ssh-slaves/target/surefire-reports/TEST-hudson.plugins.sshslaves.verifiers.VerificationStrategyConfigurationTest.xml + rm -fv pct-work/ssh-slaves/target/surefire-reports/TEST-hudson.plugins.sshslaves.SSHLauncherTest.xml +fi -# produces: pct-report.xml, **/target/surefire-reports/TEST-*.xml +# produces: **/target/surefire-reports/TEST-*.xml diff --git a/pom.xml b/pom.xml index e7a20b09e..24ae7dc74 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ ${revision}${changelist} pom - 2.176.5 + 3 -SNAPSHOT UTF-8 @@ -29,17 +29,48 @@ ${scmTag} - bom + bom-2.176.x + bom-2.164.x + bom-2.150.x + bom-2.138.x sample-plugin + + org.codehaus.mojo + flatten-maven-plugin + + + flatten + + bom + + + + maven-release-plugin bom-@{project.version} + + org.apache.maven.plugins + maven-install-plugin + false + + true + + + + org.apache.maven.plugins + maven-deploy-plugin + false + + true + + diff --git a/prep.sh b/prep.sh index 566098f53..7f09a0319 100644 --- a/prep.sh +++ b/prep.sh @@ -2,33 +2,48 @@ set -euxo pipefail cd $(dirname $0) -rm -rf sample-plugin/target - MVN='mvn -B -ntp' if [ -v MAVEN_SETTINGS ] then MVN="$MVN -s $MAVEN_SETTINGS" fi -$MVN -Dmaven.test.failure.ignore install ${SAMPLE_PLUGIN_OPTS:-} +$MVN clean install ${SAMPLE_PLUGIN_OPTS:-} -cd sample-plugin/target -cp -r jenkins-for-test megawar -mkdir jenkins -echo '# nothing' > jenkins/split-plugins.txt -jar uvf megawar/WEB-INF/lib/jenkins-core-*.jar jenkins/split-plugins.txt -rm -rfv megawar/WEB-INF/detached-plugins megawar/META-INF/*.{RSA,SF} -mkdir megawar/WEB-INF/plugins -cp -rv test-classes/test-dependencies/*.hpi megawar/WEB-INF/plugins -(cd megawar && jar c0Mf ../megawar.war *) +ALL_LINEZ=$(ls -1d bom-*.x | sort -rn | sed s/bom-//g) +: "${LINEZ:=$ALL_LINEZ}" +echo -n $LINEZ > target/lines.txt -# TODO find a way to encode this in some POM so that it can be managed by Dependabot -version=0.2.0 -pct=$HOME/.m2/repository/org/jenkins-ci/tests/plugins-compat-tester-cli/$version/plugins-compat-tester-cli-$version.jar -[ -f $pct ] || $MVN dependency:get -Dartifact=org.jenkins-ci.tests:plugins-compat-tester-cli:$version:jar -DremoteRepositories=https://repo.jenkins-ci.org/public/ -Dtransitive=false +rebuild=no +for LINE in $LINEZ +do + if [ $rebuild = yes ] + then + $MVN -f sample-plugin clean package ${SAMPLE_PLUGIN_OPTS:-} -P$LINE + else + rebuild=yes + pushd sample-plugin/target/test-classes/test-dependencies + echo -n *.hpi | sed s/.hpi//g > ../../../../target/plugins.txt + popd + fi + pushd sample-plugin/target + mkdir jenkins + echo '# nothing' > jenkins/split-plugins.txt + cp -r jenkins-for-test megawar-$LINE + jar uvf megawar-$LINE/WEB-INF/lib/jenkins-core-*.jar jenkins/split-plugins.txt + rm -rfv megawar-$LINE/WEB-INF/detached-plugins megawar-$LINE/META-INF/*.{RSA,SF} + mkdir megawar-$LINE/WEB-INF/plugins + cp -rv test-classes/test-dependencies/*.hpi megawar-$LINE/WEB-INF/plugins + cd megawar-$LINE + jar c0Mf ../../../target/megawar-$LINE.war * + popd +done -cp $pct pct.jar -cd megawar/WEB-INF/plugins -echo -n *.hpi | sed s/.hpi//g > ../../../plugins.txt +# TODO find a way to encode this in some POM so that it can be managed by Dependabot +version=0.2.1 +timestamp=20190926.172941-2 # TODO https://github.com/jenkinsci/plugin-compat-tester/pull/198 +pct=$HOME/.m2/repository/org/jenkins-ci/tests/plugins-compat-tester-cli/${version}-SNAPSHOT/plugins-compat-tester-cli-${version}-${timestamp}.jar +[ -f $pct ] || $MVN dependency:get -Dartifact=org.jenkins-ci.tests:plugins-compat-tester-cli:${version}-${timestamp}:jar -DremoteRepositories=https://repo.jenkins-ci.org/public/ -Dtransitive=false +cp $pct target/pct.jar -# produces: sample-plugin/target/{megawar.war,pct.jar,plugins.txt} +# produces: target/{megawar-*.war,pct.jar,plugins.txt,lines.txt} diff --git a/sample-plugin/check.groovy b/sample-plugin/check.groovy index fa4523e0c..63d0da6cd 100644 --- a/sample-plugin/check.groovy +++ b/sample-plugin/check.groovy @@ -16,9 +16,13 @@ def managedPluginDeps = managedDeps.collect {stripAllButGA(it)}.grep { ga -> } pluginName(art) != null } -if (managedPluginDeps != managedPluginDeps.toSorted()) { - throw new org.apache.maven.plugin.MojoFailureException("Managed plugin dependencies should be sorted: $managedPluginDeps") - // TODO also check sorting of sample plugin dependencies +if (settings.activeProfiles.any {it ==~ /^2[.][0-9]+[.]x$/}) { + println 'Skipping managed plugin dep sort check on this old LTS line' +} else { + if (managedPluginDeps != managedPluginDeps.toSorted()) { + throw new org.apache.maven.plugin.MojoFailureException("Managed plugin dependencies should be sorted: $managedPluginDeps") + // TODO also check sorting of sample plugin dependencies + } } project.artifacts.each { art -> diff --git a/sample-plugin/pom.xml b/sample-plugin/pom.xml index ee2cd5857..2d90e31b0 100644 --- a/sample-plugin/pom.xml +++ b/sample-plugin/pom.xml @@ -12,8 +12,9 @@ ${revision}${changelist} hpi - 2.176.5 + 3 -SNAPSHOT + 2.176.x 2.176.3 8 @@ -33,7 +34,7 @@ ${project.groupId} - bom + bom-${bom} ${project.version} import pom @@ -211,6 +212,13 @@ + + org.apache.maven.plugins + maven-install-plugin + + true + + org.apache.maven.plugins maven-deploy-plugin @@ -221,6 +229,27 @@ + + 2.164.x + + 2.164.x + 2.164.3 + + + + 2.150.x + + 2.150.x + 2.150.3 + + + + 2.138.x + + 2.138.x + 2.138.4 + + skip-tests-on-release