From 551a56090257448e135ec2b53eea771d315a5b26 Mon Sep 17 00:00:00 2001 From: Daniel Beck Date: Sun, 17 May 2020 00:31:02 +0200 Subject: [PATCH] [INFRA-2615] [INFRA-1021] More flexible tiering --- pom.xml | 2 +- site/generate-htaccess.sh | 93 ++++++++---- site/generate.sh | 40 +++--- .../java/io/jenkins/update_center/Main.java | 11 +- .../json/TieredUpdateSitesGenerator.java | 136 ++++++++++++++++++ 5 files changed, 232 insertions(+), 50 deletions(-) create mode 100644 src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java diff --git a/pom.xml b/pom.xml index d60075ed3..3d34efc67 100644 --- a/pom.xml +++ b/pom.xml @@ -8,7 +8,7 @@ update-center2 - 3.0-SNAPSHOT + 3.1-SNAPSHOT Jenkins Update Center Generator Generates update sites for updates.jenkins.io diff --git a/site/generate-htaccess.sh b/site/generate-htaccess.sh index 53096b32d..19c88ffcd 100755 --- a/site/generate-htaccess.sh +++ b/site/generate-htaccess.sh @@ -1,6 +1,6 @@ #!/bin/bash -USAGE="Usage: $0 [ ...] +USAGE="Usage: $0 [ ...] " [[ $# -gt 0 ]] || { echo "${USAGE}Expected at least one argument." >&2 ; exit 1 ; } @@ -15,50 +15,82 @@ cat < RewriteEngine on -# If we have a match that looks like an LTS version, e.g. 1.554.1, redirect to stable-1.554 -RewriteCond %{QUERY_STRING} ^.*version=([0-9]*\.[0-9]*)\..*$ [NC] -RewriteCond %{DOCUMENT_ROOT}/stable\-%1%{REQUEST_URI} -f - -RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable\-%1%{REQUEST_URI}? [NC,L,R=301] - EOF echo "# Version-specific rulesets generated by generate.sh" +n=$# +versions=( "$@" ) +newestLTS= +oldestLTS= + +for (( i = n-1 ; i >= 0 ; i-- )) ; do + version="${versions[i]}" + IFS=. read -ra versionPieces <<< "$version" + + major=${versionPieces[0]} + minor=${versionPieces[1]} + patch= + if [[ ${#versionPieces[@]} -gt 2 ]] ; then + patch=${versionPieces[2]} + fi + + if [[ "$version" =~ ^2[.][0-9]+[.][0-9]$ ]] ; then + # This is an LTS version + if [[ -z "$newestLTS" ]] ; then + newestLTS="$version" + fi -for ltsv in $@ ; do - v=${ltsv%.1} # support args both as '1.234' and '1.234.1'. - lastLTS=$v + cat < ${major} or major = ${major} and minor >= ${minor} or major = ${major} and minor = ${minor} and patch >= ${patch}, use this LTS update site +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 > ${major} +RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 = ${major} +RewriteCond %2 >= ${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 = ${major} +RewriteCond %2 = ${minor} +RewriteCond %3 >= ${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +EOF + oldestLTS=$version + else + # This is a weekly version # Split our version up into an array for rewriting # 1.651 becomes (1 651) - versionPieces=(${v//./ }) - major=${versionPieces[0]} - minor=${versionPieces[1]} cat < ${major} or major = ${major} and minor >= ${minor}, use this weekly update site +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC] +RewriteCond %1 >${major} +RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC] +RewriteCond %1 =${major} +RewriteCond %2 >${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301] EOF -done + fi + v=${version%.1} # support args both as '1.234' and '1.234.1'. +done -lts=$1 -versionPieces=(${lts//./ }) -major=${versionPieces[0]} -minor=${versionPieces[1]} -echo "# First LTS update site (stable-$major.$minor) gets all older releases" cat </dev/null || greadlink -f "$ echo "Main directory: $MAIN_DIR" mkdir -p "$MAIN_DIR"/tmp/ -readarray -t RELEASES < <( curl 'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1' | jq --raw-output '.results[].version' | head -n 5 | $SORT --version-sort ) || { echo "Failed to retrieve list of releases" >&2 ; exit 1 ; } +rm -rf "$MAIN_DIR"/tmp/generator/ +rm -rf "$MAIN_DIR"/tmp/generator.zip +wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.1-SNAPSHOT/update-center2-3.1-20200516.213950-1-bin.zip" +unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/ + +java -Dfile.encoding=UTF-8 -jar "$MAIN_DIR"/tmp/generator/update-center2-*.jar --dynamic-tier-list-file tmp/tiers.json +readarray -t WEEKLY_RELEASES < <( jq --raw-output '.weeklyCores[]' tmp/tiers.json ) || { echo "Failed to determine weekly tier list" >&2 ; exit 1 ; } +readarray -t STABLE_RELEASES < <( jq --raw-output '.stableCores[]' tmp/tiers.json ) || { echo "Failed to determine stable tier list" >&2 ; exit 1 ; } # prepare the www workspace for execution rm -rf "$WWW_ROOT_DIR" mkdir -p "$WWW_ROOT_DIR" # Generate htaccess file -"$( dirname "$0" )"/generate-htaccess.sh "${RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess" - -rm -rf "$MAIN_DIR"/tmp/generator/ -rm -rf "$MAIN_DIR"/tmp/generator.zip -wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.0-SNAPSHOT/update-center2-3.0-20200510.094933-31-bin.zip" -unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/ - +"$( dirname "$0" )"/generate-htaccess.sh "${WEEKLY_RELEASES[@]}" "${STABLE_RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess" # Reset arguments file echo "# one update site per line" > "$MAIN_DIR"/tmp/args.lst @@ -81,25 +82,22 @@ function sanity-check { fi } -# Generate several update sites for different segments so that plugins can +# Generate tiered update sites for different segments so that plugins can # aggressively update baseline requirements without stranding earlier users. # -# We use LTS as a boundary of different segments, to create -# a reasonable number of segments with reasonable sizes. Plugins -# tend to pick LTS baseline as the required version, so this works well. -# -# We generate tiered update sites for the five most recent LTS baselines, which -# means admins get compatible updates offered on releases up to about one year old. -for ltsv in "${RELEASES[@]}" ; do - v="${ltsv/%.1/}" - # For mainline up to $v, advertising the latest core - generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$v/latest" --www-dir "$WWW_ROOT_DIR/$v" +# We generate tiered update sites for all core releases newer than +# about 13 months that are actually used as plugin dependencies. +# This supports updating Jenkins (core) once a year while getting offered compatible plugin updates. +for version in "${WEEKLY_RELEASES[@]}" ; do + # For mainline, advertising the latest core + generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$version/latest" --www-dir "$WWW_ROOT_DIR/$version" +done +for version in "${STABLE_RELEASES[@]}" ; do # For LTS, advertising the latest LTS core - generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$v/latest" --www-dir "$WWW_ROOT_DIR/stable-$v" --only-stable-core + generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$version/latest" --www-dir "$WWW_ROOT_DIR/stable-$version" --only-stable-core done - # Experimental update center without version caps, including experimental releases. # This is not a part of the version-based redirection rules, admins need to manually configure it. # Generate this first, including --downloads-directory, as this includes all releases, experimental and otherwise. diff --git a/src/main/java/io/jenkins/update_center/Main.java b/src/main/java/io/jenkins/update_center/Main.java index 436e38832..9b2c1c062 100644 --- a/src/main/java/io/jenkins/update_center/Main.java +++ b/src/main/java/io/jenkins/update_center/Main.java @@ -27,6 +27,7 @@ import hudson.util.VersionNumber; import io.jenkins.lib.support_log_formatter.SupportLogFormatter; import io.jenkins.update_center.args4j.LevelOptionHandler; +import io.jenkins.update_center.json.TieredUpdateSitesGenerator; import io.jenkins.update_center.json.PluginDocumentationUrlsRoot; import io.jenkins.update_center.wrappers.AlphaBetaOnlyRepository; import io.jenkins.update_center.wrappers.StableWarMavenRepository; @@ -105,6 +106,9 @@ public class Main { /* Configure what kinds of output to generate */ + @Option(name = "--dynamic-tier-list-file", usage = "Generate tier list JSON file at the specified path. If this option is set, we skip generating all other output.") + @CheckForNull public File tierListFile; + @Option(name = "--www-dir", usage = "Generate simple output files, JSON(ish) and others, into this directory") @CheckForNull public File www; @@ -224,6 +228,11 @@ public void run() throws Exception { MavenRepository repo = createRepository(); + if (tierListFile != null) { + new TieredUpdateSitesGenerator().withRepository(repo).write(tierListFile, prettyPrint); + return; + } + metadataWriter.writeMetadataFiles(repo, www); if (!skipUpdateCenter) { @@ -259,7 +268,7 @@ private String updateCenterPostMessageHtml(String updateCenterJson) { private static void writeToFile(String string, final File file) throws IOException { File parentFile = file.getParentFile(); - if (!parentFile.isDirectory() && !parentFile.mkdirs()) { + if (parentFile != null && !parentFile.isDirectory() && !parentFile.mkdirs()) { throw new IOException("Failed to create parent directory " + parentFile); } PrintWriter rhpw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8)); diff --git a/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java b/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java new file mode 100644 index 000000000..f0e77f1a0 --- /dev/null +++ b/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java @@ -0,0 +1,136 @@ +/* + * The MIT License + * + * Copyright (c) 2020, Daniel Beck + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package io.jenkins.update_center.json; + +import com.alibaba.fastjson.annotation.JSONField; +import hudson.util.VersionNumber; +import io.jenkins.update_center.HPI; +import io.jenkins.update_center.JenkinsWar; +import io.jenkins.update_center.MavenRepository; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class TieredUpdateSitesGenerator extends WithoutSignature { + + private MavenRepository repository; + + @JSONField + public List weeklyCores; + + @JSONField + public List stableCores; + + public TieredUpdateSitesGenerator withRepository(MavenRepository repository) throws IOException { + this.repository = repository; + update(); + return this; + } + + private static boolean isStableVersion(VersionNumber version) { + return version.getDigitAt(2) != -1; + } + + private static VersionNumber nextWeeklyReleaseAfterStableBaseline(VersionNumber version) { + if (!version.toString().matches("[0-9][.][0-9]+[.][1-9]")) { + throw new IllegalArgumentException("Unexpected LTS version: " + version.toString()); + } + return new VersionNumber(version.getDigitAt(0) + "." + (version.getDigitAt(1) + 1)); + } + + private static boolean isReleaseRecentEnough(JenkinsWar war) { + Objects.requireNonNull(war, "war"); + return war.getTimestampAsDate().toInstant().isAfter(Instant.now().minus(CORE_AGE_DAYS, ChronoUnit.DAYS)); + } + + public void update() throws IOException { + Collection allPluginReleases = this.repository.listJenkinsPlugins().stream() + .map(plugin -> plugin.getArtifacts().values()) + .reduce(new HashSet<>(), (acc, els) -> { acc.addAll(els); return acc; }); + + final List coreDependencyVersions = allPluginReleases.stream().map(v -> { + try { + return v.getRequiredJenkinsVersion(); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to determine required Jenkins version for " + v.getGavId()); + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toSet()).stream().map(VersionNumber::new).sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + + final TreeMap allJenkinsWarsByVersionNumber = this.repository.getJenkinsWarsByVersionNumber(); + final Set weeklyCores = new HashSet<>(); + final Set stableCores = new HashSet<>(); + + boolean stableDone = false; + boolean weeklyDone = false; + + for (VersionNumber dependencyVersion : coreDependencyVersions) { + final JenkinsWar war = allJenkinsWarsByVersionNumber.get(dependencyVersion); + if (war == null) { + LOGGER.log(Level.INFO, "Did not find declared core dependency version among all core releases: " + dependencyVersion.toString()); + continue; + } + final boolean releaseRecentEnough = isReleaseRecentEnough(war); + if (isStableVersion(dependencyVersion)) { + if (!stableDone) { + if (!releaseRecentEnough) { + stableDone = true; + } + stableCores.add(dependencyVersion); + if (!weeklyDone) { + weeklyCores.add(nextWeeklyReleaseAfterStableBaseline(dependencyVersion)); + } + } + } else { + if (!weeklyDone) { + if (!releaseRecentEnough) { + weeklyDone = true; + } + weeklyCores.add(dependencyVersion); + } + } + if (stableDone && weeklyDone) { + break; + } + } + + this.stableCores = stableCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList()); + this.weeklyCores = weeklyCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList()); + } + + public static final Logger LOGGER = Logger.getLogger(TieredUpdateSitesGenerator.class.getName()); + + private static final int CORE_AGE_DAYS = 400; +}