Skip to content

Commit

Permalink
[INFRA-2615] [INFRA-1021] More flexible tiering
Browse files Browse the repository at this point in the history
  • Loading branch information
daniel-beck committed May 16, 2020
1 parent 2f7f2f1 commit 551a560
Show file tree
Hide file tree
Showing 5 changed files with 232 additions and 50 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

<artifactId>update-center2</artifactId>

<version>3.0-SNAPSHOT</version>
<version>3.1-SNAPSHOT</version>
<name>Jenkins Update Center Generator</name>
<description>Generates update sites for updates.jenkins.io</description>

Expand Down
93 changes: 66 additions & 27 deletions site/generate-htaccess.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

USAGE="Usage: $0 [<LTS baseline> ...]
USAGE="Usage: $0 [<release> ...]
"

[[ $# -gt 0 ]] || { echo "${USAGE}Expected at least one argument." >&2 ; exit 1 ; }
Expand All @@ -15,63 +15,102 @@ cat <<EOF
# See: <https://github.com/jenkinsci/backend-update-center2/blob/master/site/generate-htaccess.sh>
RewriteEngine on
# If we have a match that looks like an LTS version, e.g. 1.554.1, redirect to stable-1.554
RewriteCond %{QUERY_STRING} ^.*version=([0-9]*\.[0-9]*)\..*$ [NC]
RewriteCond %{DOCUMENT_ROOT}/stable\-%1%{REQUEST_URI} -f
RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable\-%1%{REQUEST_URI}? [NC,L,R=301]
EOF

echo "# Version-specific rulesets generated by generate.sh"
n=$#
versions=( "$@" )
newestLTS=
oldestLTS=

for (( i = n-1 ; i >= 0 ; i-- )) ; do
version="${versions[i]}"
IFS=. read -ra versionPieces <<< "$version"

major=${versionPieces[0]}
minor=${versionPieces[1]}
patch=
if [[ ${#versionPieces[@]} -gt 2 ]] ; then
patch=${versionPieces[2]}
fi

if [[ "$version" =~ ^2[.][0-9]+[.][0-9]$ ]] ; then
# This is an LTS version
if [[ -z "$newestLTS" ]] ; then
newestLTS="$version"
fi

for ltsv in $@ ; do
v=${ltsv%.1} # support args both as '1.234' and '1.234.1'.
lastLTS=$v
cat <<EOF
# If major > ${major} or major = ${major} and minor >= ${minor} or major = ${major} and minor = ${minor} and patch >= ${patch}, use this LTS update site
RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
RewriteCond %1 > ${major}
RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
RewriteCond %1 = ${major}
RewriteCond %2 >= ${minor}
RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC]
RewriteCond %1 = ${major}
RewriteCond %2 = ${minor}
RewriteCond %3 >= ${minor}
RewriteRule ^(update\-center.*\.(json|html)+) /stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301]
EOF
oldestLTS=$version
else
# This is a weekly version
# Split our version up into an array for rewriting
# 1.651 becomes (1 651)
versionPieces=(${v//./ })
major=${versionPieces[0]}
minor=${versionPieces[1]}
cat <<EOF
RewriteCond %{QUERY_STRING} ^.*version=${major}\.(\d+)$ [NC]
RewriteCond %1 <=${minor}
RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
# If major > ${major} or major = ${major} and minor >= ${minor}, use this weekly update site
RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC]
RewriteCond %1 >${major}
RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC]
RewriteCond %1 =${major}
RewriteCond %2 >${minor}
RewriteRule ^(update\-center.*\.(json|html)+) /${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
EOF

done
fi
v=${version%.1} # support args both as '1.234' and '1.234.1'.

done

lts=$1
versionPieces=(${lts//./ })
major=${versionPieces[0]}
minor=${versionPieces[1]}
echo "# First LTS update site (stable-$major.$minor) gets all older releases"
cat <<EOF
# First LTS update site (stable-$oldestLTS) gets all older releases
RewriteCond %{QUERY_STRING} ^.*version=\d\.(\d+)\.\d+$ [NC]
RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable-${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301]
RewriteRule ^(update\-center.*\.(json|html)+) /stable-${oldestLTS}%{REQUEST_URI}? [NC,L,R=301]
EOF


echo "# Add a RewriteRule for the last LTS we have, which should always rewrite to /stable"
echo "# Add a RewriteRule for /stable which will always rewrite to the last LTS site we have"
cat <<EOF
RewriteRule ^stable/(.+) "/stable-${lastLTS}/\$1" [NC,L,R=301]
RewriteRule ^stable/(.+) "/stable-${newestLTS}/\$1" [NC,L,R=301]
EOF



# Further static rules
cat <<EOF
# These are static rules
# If that all failed, but we have an update center, let's go to current
RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /current%{REQUEST_URI}? [NC,L,R=301]
RewriteRule ^(update\-center.*\.(json|html)+|latestCore\.txt) /current%{REQUEST_URI}? [NC,L,R=301]
# Ensure /release-history.json goes to the right place
RewriteRule ^release\-history\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
# Ensure /plugin-documentation-urls.json goes to the right place
RewriteRule ^plugin\-documentation\-urls\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
# Ensure /plugin-versions.json goes to the right place
RewriteRule ^plugin\-versions\.json+ /current%{REQUEST_URI}? [NC,L,R=301]
Expand All @@ -87,7 +126,7 @@ RewriteCond %{HTTPS} !=on
RewriteRule (.*\.json(\.html)?)$ http://mirrors.jenkins-ci.org/updates/\$1
# TODO this might be unnecessary?
# download/* directories contain virtual URL spaces for redirecting download traffic to mirrors.
RedirectMatch 302 /download/war/([0-9]*\.[0-9]*\.[0-9]*/jenkins)\.war$ http://mirrors.jenkins-ci.org/war-stable/\$1.war
RedirectMatch 302 /download/war/(.*)\.war$ http://mirrors.jenkins-ci.org/war/\$1.war
Expand Down
40 changes: 19 additions & 21 deletions site/generate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,20 +50,21 @@ MAIN_DIR="$( readlink -f "$SIMPLE_SCRIPT_DIR/../" 2>/dev/null || greadlink -f "$
echo "Main directory: $MAIN_DIR"
mkdir -p "$MAIN_DIR"/tmp/

readarray -t RELEASES < <( curl 'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1' | jq --raw-output '.results[].version' | head -n 5 | $SORT --version-sort ) || { echo "Failed to retrieve list of releases" >&2 ; exit 1 ; }
rm -rf "$MAIN_DIR"/tmp/generator/
rm -rf "$MAIN_DIR"/tmp/generator.zip
wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.1-SNAPSHOT/update-center2-3.1-20200516.213950-1-bin.zip"
unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/

java -Dfile.encoding=UTF-8 -jar "$MAIN_DIR"/tmp/generator/update-center2-*.jar --dynamic-tier-list-file tmp/tiers.json
readarray -t WEEKLY_RELEASES < <( jq --raw-output '.weeklyCores[]' tmp/tiers.json ) || { echo "Failed to determine weekly tier list" >&2 ; exit 1 ; }
readarray -t STABLE_RELEASES < <( jq --raw-output '.stableCores[]' tmp/tiers.json ) || { echo "Failed to determine stable tier list" >&2 ; exit 1 ; }

# prepare the www workspace for execution
rm -rf "$WWW_ROOT_DIR"
mkdir -p "$WWW_ROOT_DIR"

# Generate htaccess file
"$( dirname "$0" )"/generate-htaccess.sh "${RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess"

rm -rf "$MAIN_DIR"/tmp/generator/
rm -rf "$MAIN_DIR"/tmp/generator.zip
wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.0-SNAPSHOT/update-center2-3.0-20200510.094933-31-bin.zip"
unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/

"$( dirname "$0" )"/generate-htaccess.sh "${WEEKLY_RELEASES[@]}" "${STABLE_RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess"

# Reset arguments file
echo "# one update site per line" > "$MAIN_DIR"/tmp/args.lst
Expand All @@ -81,25 +82,22 @@ function sanity-check {
fi
}

# Generate several update sites for different segments so that plugins can
# Generate tiered update sites for different segments so that plugins can
# aggressively update baseline requirements without stranding earlier users.
#
# We use LTS as a boundary of different segments, to create
# a reasonable number of segments with reasonable sizes. Plugins
# tend to pick LTS baseline as the required version, so this works well.
#
# We generate tiered update sites for the five most recent LTS baselines, which
# means admins get compatible updates offered on releases up to about one year old.
for ltsv in "${RELEASES[@]}" ; do
v="${ltsv/%.1/}"
# For mainline up to $v, advertising the latest core
generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$v/latest" --www-dir "$WWW_ROOT_DIR/$v"
# We generate tiered update sites for all core releases newer than
# about 13 months that are actually used as plugin dependencies.
# This supports updating Jenkins (core) once a year while getting offered compatible plugin updates.
for version in "${WEEKLY_RELEASES[@]}" ; do
# For mainline, advertising the latest core
generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/$version/latest" --www-dir "$WWW_ROOT_DIR/$version"
done

for version in "${STABLE_RELEASES[@]}" ; do
# For LTS, advertising the latest LTS core
generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$v/latest" --www-dir "$WWW_ROOT_DIR/stable-$v" --only-stable-core
generate --limit-plugin-core-dependency "$version.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$version/latest" --www-dir "$WWW_ROOT_DIR/stable-$version" --only-stable-core
done


# Experimental update center without version caps, including experimental releases.
# This is not a part of the version-based redirection rules, admins need to manually configure it.
# Generate this first, including --downloads-directory, as this includes all releases, experimental and otherwise.
Expand Down
11 changes: 10 additions & 1 deletion src/main/java/io/jenkins/update_center/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import hudson.util.VersionNumber;
import io.jenkins.lib.support_log_formatter.SupportLogFormatter;
import io.jenkins.update_center.args4j.LevelOptionHandler;
import io.jenkins.update_center.json.TieredUpdateSitesGenerator;
import io.jenkins.update_center.json.PluginDocumentationUrlsRoot;
import io.jenkins.update_center.wrappers.AlphaBetaOnlyRepository;
import io.jenkins.update_center.wrappers.StableWarMavenRepository;
Expand Down Expand Up @@ -105,6 +106,9 @@ public class Main {


/* Configure what kinds of output to generate */
@Option(name = "--dynamic-tier-list-file", usage = "Generate tier list JSON file at the specified path. If this option is set, we skip generating all other output.")
@CheckForNull public File tierListFile;

@Option(name = "--www-dir", usage = "Generate simple output files, JSON(ish) and others, into this directory")
@CheckForNull public File www;

Expand Down Expand Up @@ -224,6 +228,11 @@ public void run() throws Exception {

MavenRepository repo = createRepository();

if (tierListFile != null) {
new TieredUpdateSitesGenerator().withRepository(repo).write(tierListFile, prettyPrint);
return;
}

metadataWriter.writeMetadataFiles(repo, www);

if (!skipUpdateCenter) {
Expand Down Expand Up @@ -259,7 +268,7 @@ private String updateCenterPostMessageHtml(String updateCenterJson) {

private static void writeToFile(String string, final File file) throws IOException {
File parentFile = file.getParentFile();
if (!parentFile.isDirectory() && !parentFile.mkdirs()) {
if (parentFile != null && !parentFile.isDirectory() && !parentFile.mkdirs()) {
throw new IOException("Failed to create parent directory " + parentFile);
}
PrintWriter rhpw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,136 @@
/*
* The MIT License
*
* Copyright (c) 2020, Daniel Beck
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package io.jenkins.update_center.json;

import com.alibaba.fastjson.annotation.JSONField;
import hudson.util.VersionNumber;
import io.jenkins.update_center.HPI;
import io.jenkins.update_center.JenkinsWar;
import io.jenkins.update_center.MavenRepository;

import java.io.IOException;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.TreeMap;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;

public class TieredUpdateSitesGenerator extends WithoutSignature {

private MavenRepository repository;

@JSONField
public List<String> weeklyCores;

@JSONField
public List<String> stableCores;

public TieredUpdateSitesGenerator withRepository(MavenRepository repository) throws IOException {
this.repository = repository;
update();
return this;
}

private static boolean isStableVersion(VersionNumber version) {
return version.getDigitAt(2) != -1;
}

private static VersionNumber nextWeeklyReleaseAfterStableBaseline(VersionNumber version) {
if (!version.toString().matches("[0-9][.][0-9]+[.][1-9]")) {
throw new IllegalArgumentException("Unexpected LTS version: " + version.toString());
}
return new VersionNumber(version.getDigitAt(0) + "." + (version.getDigitAt(1) + 1));
}

private static boolean isReleaseRecentEnough(JenkinsWar war) {
Objects.requireNonNull(war, "war");
return war.getTimestampAsDate().toInstant().isAfter(Instant.now().minus(CORE_AGE_DAYS, ChronoUnit.DAYS));
}

public void update() throws IOException {
Collection<HPI> allPluginReleases = this.repository.listJenkinsPlugins().stream()
.map(plugin -> plugin.getArtifacts().values())
.reduce(new HashSet<>(), (acc, els) -> { acc.addAll(els); return acc; });

final List<VersionNumber> coreDependencyVersions = allPluginReleases.stream().map(v -> {
try {
return v.getRequiredJenkinsVersion();
} catch (IOException e) {
LOGGER.log(Level.WARNING, "Failed to determine required Jenkins version for " + v.getGavId());
return null;
}
}).filter(Objects::nonNull).collect(Collectors.toSet()).stream().map(VersionNumber::new).sorted(Comparator.reverseOrder()).collect(Collectors.toList());

final TreeMap<VersionNumber, JenkinsWar> allJenkinsWarsByVersionNumber = this.repository.getJenkinsWarsByVersionNumber();
final Set<VersionNumber> weeklyCores = new HashSet<>();
final Set<VersionNumber> stableCores = new HashSet<>();

boolean stableDone = false;
boolean weeklyDone = false;

for (VersionNumber dependencyVersion : coreDependencyVersions) {
final JenkinsWar war = allJenkinsWarsByVersionNumber.get(dependencyVersion);
if (war == null) {
LOGGER.log(Level.INFO, "Did not find declared core dependency version among all core releases: " + dependencyVersion.toString());
continue;
}
final boolean releaseRecentEnough = isReleaseRecentEnough(war);
if (isStableVersion(dependencyVersion)) {
if (!stableDone) {
if (!releaseRecentEnough) {
stableDone = true;
}
stableCores.add(dependencyVersion);
if (!weeklyDone) {
weeklyCores.add(nextWeeklyReleaseAfterStableBaseline(dependencyVersion));
}
}
} else {
if (!weeklyDone) {
if (!releaseRecentEnough) {
weeklyDone = true;
}
weeklyCores.add(dependencyVersion);
}
}
if (stableDone && weeklyDone) {
break;
}
}

this.stableCores = stableCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList());
this.weeklyCores = weeklyCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList());
}

public static final Logger LOGGER = Logger.getLogger(TieredUpdateSitesGenerator.class.getName());

private static final int CORE_AGE_DAYS = 400;
}

0 comments on commit 551a560

Please sign in to comment.