[INFRA-2615] [INFRA-1021] More flexible tiering

pom.xml

@@ -8,7 +8,7 @@
 
   <artifactId>update-center2</artifactId>
 
-  <version>3.0-SNAPSHOT</version>
+  <version>3.1-SNAPSHOT</version>
   <name>Jenkins Update Center Generator</name>
   <description>Generates update sites for updates.jenkins.io</description>
 

site/generate-htaccess.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-USAGE="Usage: $0 [<LTS baseline> ...]
+USAGE="Usage: $0 [<release> ...]
 "
 
 [[ $# -gt 0 ]] || { echo "${USAGE}Expected at least one argument." >&2 ; exit 1 ; }
@@ -15,63 +15,102 @@ cat <<EOF
 # See: <https://github.com/jenkinsci/backend-update-center2/blob/master/site/generate-htaccess.sh>
 RewriteEngine on
 
-# If we have a match that looks like an LTS version, e.g. 1.554.1, redirect to stable-1.554
-RewriteCond %{QUERY_STRING} ^.*version=([0-9]*\.[0-9]*)\..*$ [NC]
-RewriteCond %{DOCUMENT_ROOT}/stable\-%1%{REQUEST_URI} -f
-
-RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable\-%1%{REQUEST_URI}? [NC,L,R=301]
-
 EOF
 
 echo "# Version-specific rulesets generated by generate.sh"
+n=$#
+versions=( "$@" )
+newestLTS=
+oldestLTS=
+
+for (( i = n-1 ; i >= 0 ; i-- )) ; do
+  version="${versions[i]}"
+  IFS=. read -ra versionPieces <<< "$version"
+
+  major=${versionPieces[0]}
+  minor=${versionPieces[1]}
+  patch=
+  if [[ ${#versionPieces[@]} -gt 2 ]] ; then
+    patch=${versionPieces[2]}
+  fi
+
+  if [[ "$version" =~ ^2[.][0-9]+[.][0-9]$ ]] ; then
+    # This is an LTS version
+    if [[ -z "$newestLTS" ]] ; then
+      newestLTS="$version"
+    fi
 
-for ltsv in $@ ; do
-    v=${ltsv%.1} # support args both as '1.234' and '1.234.1'.
-    lastLTS=$v
+    cat <<EOF
 
+# If major > ${major} or major = ${major} and minor >= ${minor} or major = ${major} and minor = ${minor} and patch >= ${patch}, use this LTS update site
+RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
+RewriteCond %1 > ${major}
+RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
+RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
+RewriteCond %1 = ${major}
+RewriteCond %2 >= ${minor}
+RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
+RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
+RewriteCond %1 = ${major}
+RewriteCond %2 = ${minor}
+RewriteCond %3 >= ${minor}
+RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
+EOF
+    oldestLTS=$version
+  else
+    # This is a weekly version
     # Split our version up into an array for rewriting
     # 1.651 becomes (1 651)
-    versionPieces=(${v//./ })
-    major=${versionPieces[0]}
-    minor=${versionPieces[1]}
     cat <<EOF
-RewriteCond %{QUERY_STRING} ^.*version=${major}\.(\d+)$ [NC]
-RewriteCond %1 <=${minor}
-RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
 
+# If major > ${major} or major = ${major} and minor >= ${minor}, use this weekly update site
+RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC]
+RewriteCond %1 >${major}
+RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
+RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC]
+RewriteCond %1 =${major}
+RewriteCond %2 >${minor}
+RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
 EOF
 
-done
+  fi
+  v=${version%.1} # support args both as '1.234' and '1.234.1'.
 
+done
 
-lts=$1
-versionPieces=(${lts//./ })
-major=${versionPieces[0]}
-minor=${versionPieces[1]}
-echo "# First LTS update site (stable-$major.$minor) gets all older releases"
 cat <<EOF
+
+# First LTS update site (stable-$oldestLTS) gets all older releases
+
 RewriteCond %{QUERY_STRING} ^.*version=\d\.(\d+)\.\d+$ [NC]
-RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable-${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
+RewriteRule ^(update\-center.*\.(json|html)+) /stable-${oldestLTS}%{REQUEST_URI}? [NC,L,R=301]
 
 EOF
 
 
-echo "# Add a RewriteRule for the last LTS we have, which should always rewrite to /stable"
+echo "# Add a RewriteRule for /stable which will always rewrite to the last LTS site we have"
 cat <<EOF
-RewriteRule ^stable/(.+) "/stable-${lastLTS}/\$1" [NC,L,R=301]
+RewriteRule ^stable/(.+) "/stable-${newestLTS}/\$1" [NC,L,R=301]
 
 EOF
 
 
 
 # Further static rules
 cat <<EOF
+
+
+# These are static rules
+
 # If that all failed, but we have an update center, let's go to current
-RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /current%{REQUEST_URI}? [NC,L,R=301]
+RewriteRule ^(update\-center.*\.(json|html)+|latestCore\.txt) /current%{REQUEST_URI}? [NC,L,R=301]
 
 # Ensure /release-history.json goes to the right place
 RewriteRule ^release\-history\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
 
+# Ensure /plugin-documentation-urls.json goes to the right place
+RewriteRule ^plugin\-documentation\-urls\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
+
 # Ensure /plugin-versions.json goes to the right place
 RewriteRule ^plugin\-versions\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
 
@@ -87,7 +126,7 @@ RewriteCond %{HTTPS} !=on
 RewriteRule (.*\.json(\.html)?)$ http://mirrors.jenkins-ci.org/updates/\$1
 
 
-
+# TODO this might be unnecessary?
 # download/* directories contain virtual URL spaces for redirecting download traffic to mirrors.
 RedirectMatch 302 /download/war/([0-9]*\.[0-9]*\.[0-9]*/jenkins)\.war$ http://mirrors.jenkins-ci.org/war-stable/\$1.war
 RedirectMatch 302 /download/war/(.*)\.war$ http://mirrors.jenkins-ci.org/war/\$1.war

site/generate.sh

@@ -50,20 +50,21 @@ MAIN_DIR="$( readlink -f "$SIMPLE_SCRIPT_DIR/../" 2>/dev/null || greadlink -f "$
 echo "Main directory: $MAIN_DIR"
 mkdir -p "$MAIN_DIR"/tmp/
 
-readarray -t RELEASES < <( curl 'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1' | jq --raw-output '.results[].version' | head -n 5 | $SORT --version-sort ) || { echo "Failed to retrieve list of releases" >&2 ; exit 1 ; }
+rm -rf "$MAIN_DIR"/tmp/generator/
+rm -rf "$MAIN_DIR"/tmp/generator.zip
+wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.1-SNAPSHOT/update-center2-3.1-20200516.213950-1-bin.zip"
+unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/
+
+java -Dfile.encoding=UTF-8 -jar "$MAIN_DIR"/tmp/generator/update-center2-*.jar --dynamic-tier-list-file tmp/tiers.json
+readarray -t WEEKLY_RELEASES < <( jq --raw-output '.weeklyCores[]' tmp/tiers.json ) || { echo "Failed to determine weekly tier list" >&2 ; exit 1 ; }
+readarray -t STABLE_RELEASES < <( jq --raw-output '.stableCores[]' tmp/tiers.json ) || { echo "Failed to determine stable tier list" >&2 ; exit 1 ; }
 
 # prepare the www workspace for execution
 rm -rf "$WWW_ROOT_DIR"
 mkdir -p "$WWW_ROOT_DIR"
 
 # Generate htaccess file
-"$( dirname "$0" )"/generate-htaccess.sh "${RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess"
-
-rm -rf "$MAIN_DIR"/tmp/generator/
-rm -rf "$MAIN_DIR"/tmp/generator.zip
-wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.0-SNAPSHOT/update-center2-3.0-20200510.094933-31-bin.zip"
-unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/
-
+"$( dirname "$0" )"/generate-htaccess.sh "${WEEKLY_RELEASES[@]}" "${STABLE_RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess"
 
 # Reset arguments file
 echo "# one update site per line" > "$MAIN_DIR"/tmp/args.lst
@@ -81,25 +82,22 @@ function sanity-check {
   fi
 }
 
-# Generate several update sites for different segments so that plugins can
+# Generate tiered update sites for different segments so that plugins can
 # aggressively update baseline requirements without stranding earlier users.
 #
-# We use LTS as a boundary of different segments, to create
-# a reasonable number of segments with reasonable sizes. Plugins
-# tend to pick LTS baseline as the required version, so this works well.
-#
-# We generate tiered update sites for the five most recent LTS baselines, which
-# means admins get compatible updates offered on releases up to about one year old.
-for ltsv in "${RELEASES[@]}" ; do
-  v="${ltsv/%.1/}"
-  # For mainline up to $v, advertising the latest core
-  generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$v/latest" --www-dir "$WWW_ROOT_DIR/$v"
+# We generate tiered update sites for all core releases newer than
+# about 13 months that are actually used as plugin dependencies.
+# This supports updating Jenkins (core) once a year while getting offered compatible plugin updates.
+for version in "${WEEKLY_RELEASES[@]}" ; do
+  # For mainline, advertising the latest core
+  generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$version/latest" --www-dir "$WWW_ROOT_DIR/$version"
+done
 
+for version in "${STABLE_RELEASES[@]}" ; do
   # For LTS, advertising the latest LTS core
-  generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$v/latest" --www-dir "$WWW_ROOT_DIR/stable-$v" --only-stable-core
+  generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$version/latest" --www-dir "$WWW_ROOT_DIR/stable-$version" --only-stable-core
 done
 
-
 # Experimental update center without version caps, including experimental releases.
 # This is not a part of the version-based redirection rules, admins need to manually configure it.
 # Generate this first, including --downloads-directory, as this includes all releases, experimental and otherwise.

src/main/java/io/jenkins/update_center/Main.java

@@ -27,6 +27,7 @@
 import hudson.util.VersionNumber;
 import io.jenkins.lib.support_log_formatter.SupportLogFormatter;
 import io.jenkins.update_center.args4j.LevelOptionHandler;
+import io.jenkins.update_center.json.TieredUpdateSitesGenerator;
 import io.jenkins.update_center.json.PluginDocumentationUrlsRoot;
 import io.jenkins.update_center.wrappers.AlphaBetaOnlyRepository;
 import io.jenkins.update_center.wrappers.StableWarMavenRepository;
@@ -56,6 +57,7 @@
 import java.lang.reflect.Modifier;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 import java.util.logging.ConsoleHandler;
@@ -105,6 +107,9 @@ public class Main {
 
 
     /* Configure what kinds of output to generate */
+    @Option(name = "--dynamic-tier-list-file", usage = "Generate tier list JSON file at the specified path. If this option is set, we skip generating all other output.")
+    @CheckForNull public File tierListFile;
+
     @Option(name = "--www-dir", usage = "Generate simple output files, JSON(ish) and others, into this directory")
     @CheckForNull public File www;
 
@@ -158,9 +163,12 @@ public static void main(String[] args) throws Exception {
 
     public int run(String[] args) throws Exception {
         CmdLineParser p = new CmdLineParser(this);
-        new ClassParser().parse(signer, p);
-        new ClassParser().parse(metadataWriter, p);
-        new ClassParser().parse(directoryTreeBuilder, p);
+        List<Object> optionHolders = Arrays.asList(signer, metadataWriter, directoryTreeBuilder);
+
+        for (Object o : optionHolders) {
+            new ClassParser().parse(o, p);
+        }
+
         try {
             p.parseArgument(args);
 
@@ -224,6 +232,11 @@ public void run() throws Exception {
 
         MavenRepository repo = createRepository();
 
+        if (tierListFile != null) {
+            new TieredUpdateSitesGenerator().withRepository(repo).write(tierListFile, prettyPrint);
+            return;
+        }
+
         metadataWriter.writeMetadataFiles(repo, www);
 
         if (!skipUpdateCenter) {
@@ -259,7 +272,7 @@ private String updateCenterPostMessageHtml(String updateCenterJson) {
 
     private static void writeToFile(String string, final File file) throws IOException {
         File parentFile = file.getParentFile();
-        if (!parentFile.isDirectory() && !parentFile.mkdirs()) {
+        if (parentFile != null && !parentFile.isDirectory() && !parentFile.mkdirs()) {
             throw new IOException("Failed to create parent directory " + parentFile);
         }
         PrintWriter rhpw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8));