From 5526acc35ea88f4748d30166dde193e003ed7c2c Mon Sep 17 00:00:00 2001 From: Daniel Beck <1831569+daniel-beck@users.noreply.github.com> Date: Thu, 25 Jun 2020 14:57:06 +0200 Subject: [PATCH] [INFRA-2615] [INFRA-1021] More flexible tiering (#376) Co-authored-by: Daniel Beck --- README.adoc | 25 +- site/LAYOUT.md | 30 + site/README.md | 60 +- site/generate-htaccess.sh | 99 +++- site/generate.sh | 53 +- site/static/readme.html | 6 + site/test/.gitignore | 1 + site/test/Dockerfile | 7 + site/test/httpd.conf | 551 ++++++++++++++++++ site/test/redirects.sh | 61 -- site/test/test.sh | 83 +++ .../java/io/jenkins/update_center/Main.java | 11 +- .../json/TieredUpdateSitesGenerator.java | 136 +++++ 13 files changed, 976 insertions(+), 147 deletions(-) create mode 100644 site/test/.gitignore create mode 100644 site/test/Dockerfile create mode 100644 site/test/httpd.conf delete mode 100755 site/test/redirects.sh create mode 100755 site/test/test.sh create mode 100644 src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java diff --git a/README.adoc b/README.adoc index e515d879a..f177efeb0 100644 --- a/README.adoc +++ b/README.adoc @@ -126,11 +126,11 @@ These warnings are defined in the file `resources/warnings.json`. Build (`mvn clean verify`) the generator and then invoke it as follows: - java -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --id default ... + java -Dfile.encoding=UTF-8 -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --id default ... The tool also supports batch mode execution, generating multiple update sites with a single invocation: - java -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --arguments-file + java -Dfile.encoding=UTF-8 -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --arguments-file `filename.txt` is a text file with a list of arguments on each line. Lines that start with `#` are comments and ignored. @@ -157,7 +157,7 @@ Example: For a full list of arguments, invoke the tool as follows: - java -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --help + java -Dfile.encoding=UTF-8 -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --help NOTE: `--help` isn't a real argument, but usage instructions are printed when an invalid argument is provided. @@ -170,8 +170,15 @@ The following steps are therefore useful when trying to generate output correspo 1. Implement changes in `src/main/`. 2. Run `./site/generate.sh` until the Java tool is actually launched, then abort. This requires some environment variables to be defined. 3. Edit `tmp/args.lst`, changing or removing the `--key`, `--certificate`, and `--root-certificate` arguments as necessary. -4. Run `+java -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --arguments-file tmp/args.lst+` +4. Run `+java -Dfile.encoding=UTF-8 -jar target/update-center2-*-SNAPSHOT-bin/update-center2-*-SNAPSHOT.jar --arguments-file tmp/args.lst+` +Alternatively, the closest you can get to real executions in local development: + +1. Implement changes in `src/main/`. +2. Deploy a snapshot using `mvn deploy`. Requires an account in the Jenkins project, see _Deploying changes_ below. +3. Edit `site/generate.sh` to reference the specific snapshot you deployed (including timestamp) where it is downloaded using `wget`, see previous build output. +4. Optionally, to speed things up, edit `site/generate.sh` and remove the arguments `--downloads-directory "$DOWNLOAD_ROOT_DIR"` from some of the invocations. +5. Run `./site/generate.sh `. The first argument is the output directory for metadata, the second argument is the output directory for downloads and unused unless the previous step 4 was skipped. === Running within an IDE @@ -198,6 +205,16 @@ A new release (or at minimum a snapshot deployment) is needed, which is then ref NOTE: As of May 2020, everyone can deploy snapshots to Artifactory, so permissions issues shouldn't hinder development. +=== Working with htaccess/mod_rewrite rules + +The wrapper script `site/generate.sh` calls the script `site/generate-htaccess.sh` with chosen arguments. +The latter script will generate the `.htaccess` file mostly containing mod_rewrite rules to redirect requests to appropriate tiered update sites. +To learn more about tiers, see link:site/LAYOUT.md[LAYOUT.md]. + +To test changes to `site/generate-htaccess.sh`, run `site/test/test.sh`. +It executes `site/generate-htaccess.sh` and places it inside an Apache HTTPD Docker container and tests whether redirect rules are correctly applied. + + === Working with certificates To sign JSON output files, create a development certificate: diff --git a/site/LAYOUT.md b/site/LAYOUT.md index e9d388c01..530e92a93 100644 --- a/site/LAYOUT.md +++ b/site/LAYOUT.md @@ -196,3 +196,33 @@ The top-level `.htaccess` file is created by `generate-htaccess.sh` and implemen * Redirects from top-level release history and plugin versions JSON files to the real files in `current` * Redirects for any other `.json` / `.json.html` files (i.e. tool downloader metadata in `updates/`) to the mirrors network. * Redirects for `.war` and `.hpi` files in the `download/` directory tree to the mirrors network. + + +## Known consumers + +The following is a list of known references to specific files or directories in the Jenkins project. +It is intended to ensure that Jenkins to update-center2 will not break other parts of project infrastructure. + +This list is current as of June 2020, but may not be complete. + +### jenkins-infra/jenkins.io + +* `/latestCore.txt` in [fetch-external-resources](https://github.com/jenkins-infra/jenkins.io/blob/6eddbd1e2891a39a88f04387d2aea9f23bb2bdf1/scripts/fetch-external-resources#L18) +* `/stable/latestCore.txt` in [fetch-external-resources](https://github.com/jenkins-infra/jenkins.io/blob/6eddbd1e2891a39a88f04387d2aea9f23bb2bdf1/scripts/fetch-external-resources#L24) +* `/update-center.actual.json` in [fetch-external-resources](https://github.com/jenkins-infra/jenkins.io/blob/6eddbd1e2891a39a88f04387d2aea9f23bb2bdf1/scripts/fetch-external-resources#L48) +* `/release-history.json` in [Makefile](https://github.com/jenkins-infra/jenkins.io/blob/6eddbd1e2891a39a88f04387d2aea9f23bb2bdf1/Makefile#L46) + +### jenkins-infra/plugin-site-api + +* `/current/plugin-documentation-urls.json` in [WikiPluginDataParser.java](https://github.com/jenkins-infra/plugin-site-api/blob/290e6117eec06a70e2652e6270f26c3ab6e7058e/src/main/java/io/jenkins/plugins/generate/parsers/WikiPluginDataParser.java#L30) + +### jenkinsci/plugin-installation-manager-tool + +* `/update-center.actual.json` in [Settings.java](https://github.com/jenkinsci/plugin-installation-manager-tool/blob/master/plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/config/Settings.java#L13) +* `/experimental/update-center.actual.json` in [Settings.java](https://github.com/jenkinsci/plugin-installation-manager-tool/blob/master/plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/config/Settings.java#L15) +* `/current/plugin-versions.json` in [Settings.java](https://github.com/jenkinsci/plugin-installation-manager-tool/blob/master/plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/config/Settings.java#L18) + +### jenkinsci/docker + +* LTS tiered update sites per https://github.com/jenkinsci/docker/issues/954 +* Tiered `latest/` directories per https://github.com/jenkinsci/docker/issues/953 diff --git a/site/README.md b/site/README.md index cd7403c27..55fdd1f57 100644 --- a/site/README.md +++ b/site/README.md @@ -1,61 +1,45 @@ # OSS Update Center Site Architecture This script generate the code and data behind https://updates.jenkins-ci.org/ -The service this website provides is as follows. +The service this website provides is as follows: 1. Jenkins will hit well-known URLs hard-coded into Jenkins binaries with the Jenkins version number attached as a query string, like - http://updates.jenkins-ci.org/update-center.json.html?version=1.345 + https://updates.jenkins.io/update-center.json?version=1.345 1. We use the version number to redirect the traffic to the right update site, among all the ones that we generate. - 1. HTTP traffic will be directed to mirrors through http://mirror.jenkins-ci.org/ - HTTPS traffic will be served locally because we don't have any HTTPS mirrors. - In this way, Jenkins running in HTTPS will not see the insecure content warning - in the browser. ## Multiple update sites for different version ranges -Update center metadata can contain only one version per a plugin. -Because newer versions of the same plugin may depend on newer version of Jenkins, -if we just serve one update center for every Jenkins out there, older versions of -Jenkins will see plugin versions that do not work with them, making it impossible to install -the said plugin. This is unfortunate because some younger versions of the plugin might have -worked with that Jenkins core. This creates a disincentive for plugin developers -to move to the new base version, and slows down the pace in which it adopts new core features. - -So in this "v3" UC, we generate several update centers targeted for different -version ranges. Consider the following example: - - +### Dynamic update site tiers - 1.532 1.554 1.565 - -----------+-----------+-----------+---------------> 1.590 - \ \ \ - \ \ \ - ---------> ---------> -----------> +Update center metadata can contain only one version per a plugin. +Because newer versions of the same plugin may depend on newer version of Jenkins, if we just serve one update center for every Jenkins out there, older versions of Jenkins will see plugin versions that do not work with them, making it impossible to install the said plugin. +This is unfortunate because some younger versions of the plugin might have worked with that Jenkins core. +This creates a disincentive for plugin developers to move to the new base version, and slows down the pace in which it adopts new core features. +So we generate several update centers targeted for different version ranges. -The version range is defined based on LTS branch points. We pick up 3 or 4 recent -LTS branch points to create 6 or 8 ranges. +To accommodate all recent Jenkins releases, we first inspect all plugin releases for their Jenkins core dependencies. +We then generate tiered update sites for all releases identified this way that are more recent than a cutoff (~3 months). +Directories containing these tiered update sites have the prefix `dynamic-`. - * version in the range (,1.532] will see the UC that advertises 1.590 as the core release, - and they will only see plugins that are compatible with 1.532 or earlier. +mod_rewrite rules in an `.htaccess` file are then used to redirect requests from Jenkins versions to the next lower update site. +It will serve the newest release of each plugin that is compatible with the specified Jenkins version. +See [generate-htaccess.sh](generate-htaccess.sh) for how these rules are generated. - * next, version range in (1.532, 1.532.*], which is basically 1.532.x LTS, will see - the UC that advertises 1.532.x as the core release, and only see plugins that are - compatible with 1.532.* or earlier. +### Static update site tiers - * next, version in the range (1.532, 1.554] will see 1.590 as the core, and - only see plugins compatible with 1.554 or earlier. +Before June 2020, update site tiers were fixed: +Update sites were generated for the five most recent LTS baselines, and each update site would offer plugins compatible with the latest release of each LTS line. - * next, version range is (1,554, 1.554.*] and you get the idea +These update site tiers are *deprecated*: They are currently still being generated, but `.htaccess` no longer reference them. +See e.g. https://github.com/jenkinsci/docker/issues/954 - * finally, the catch-all update center advertises 1.590 core release and latest plugins - regardless of their required core versions. This applies to (1.565,*] -## Redirection logic +## Generating update sites [generate.sh](generate.sh) is run by [a CI job](https://trusted.ci.jenkins.io/job/update_center/) and generates all the different sites as static files, and deploys the directory into Apache. @@ -63,5 +47,7 @@ and generates all the different sites as static files, and deploys the directory A part of this is [.htaccess](static/.htaccess) that uses `mod_rewrite` to redirect inbound requests to the right version specific website. + ## Layout -See [a separate doc](LAYOUT.md) for the layout of the generated update site. \ No newline at end of file + +See [a separate doc](LAYOUT.md) for the layout of the generated update site. diff --git a/site/generate-htaccess.sh b/site/generate-htaccess.sh index 53096b32d..e3a9b24ab 100755 --- a/site/generate-htaccess.sh +++ b/site/generate-htaccess.sh @@ -1,6 +1,6 @@ #!/bin/bash -USAGE="Usage: $0 [ ...] +USAGE="Usage: $0 [ ...] " [[ $# -gt 0 ]] || { echo "${USAGE}Expected at least one argument." >&2 ; exit 1 ; } @@ -12,53 +12,89 @@ set -o nounset cat < +# See: RewriteEngine on -# If we have a match that looks like an LTS version, e.g. 1.554.1, redirect to stable-1.554 -RewriteCond %{QUERY_STRING} ^.*version=([0-9]*\.[0-9]*)\..*$ [NC] -RewriteCond %{DOCUMENT_ROOT}/stable\-%1%{REQUEST_URI} -f - -RewriteRule ^(update\-center.*\.(json|html)+|plugin\-documentation\-urls\.json|latestCore\.txt) /stable\-%1%{REQUEST_URI}? [NC,L,R=301] - EOF echo "# Version-specific rulesets generated by generate.sh" +n=$# +versions=( "$@" ) +newestStable= +oldestStable= +oldestWeekly= + +for (( i = n-1 ; i >= 0 ; i-- )) ; do + version="${versions[i]}" + IFS=. read -ra versionPieces <<< "$version" + + major=${versionPieces[0]} + minor=${versionPieces[1]} + patch= + if [[ ${#versionPieces[@]} -gt 2 ]] ; then + patch=${versionPieces[2]} + fi + + if [[ "$version" =~ ^2[.][0-9]+[.][0-9]$ ]] ; then + # This is an LTS version + if [[ -z "$newestStable" ]] ; then + newestStable="$version" + fi -for ltsv in $@ ; do - v=${ltsv%.1} # support args both as '1.234' and '1.234.1'. - lastLTS=$v + cat < ${major} or major = ${major} and minor >= ${minor} or major = ${major} and minor = ${minor} and patch >= ${patch}, use this LTS update site +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 >${major} +RewriteRule ^(update\-center.*\.(json|html)+) /dynamic-stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 =${major} +RewriteCond %2 >=${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /dynamic-stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)\.(\d+)$ [NC] +RewriteCond %1 =${major} +RewriteCond %2 =${minor} +RewriteCond %3 >=${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /dynamic-stable-${major}\.${minor}\.${patch}%{REQUEST_URI}? [NC,L,R=301] +EOF + oldestStable="$version" + else + # This is a weekly version # Split our version up into an array for rewriting # 1.651 becomes (1 651) - versionPieces=(${v//./ }) - major=${versionPieces[0]} - minor=${versionPieces[1]} + oldestWeekly="$version" cat < ${major} or major = ${major} and minor >= ${minor}, use this weekly update site +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC] +RewriteCond %1 >${major} +RewriteRule ^(update\-center.*\.(json|html)+) /dynamic-${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301] +RewriteCond %{QUERY_STRING} ^.*version=(\d)\.(\d+)$ [NC] +RewriteCond %1 =${major} +RewriteCond %2 >${minor} +RewriteRule ^(update\-center.*\.(json|html)+) /dynamic-${major}\.${minor}%{REQUEST_URI}? [NC,L,R=301] EOF + fi done - -lts=$1 -versionPieces=(${lts//./ }) -major=${versionPieces[0]} -minor=${versionPieces[1]} -echo "# First LTS update site (stable-$major.$minor) gets all older releases" cat </dev/null || greadlink -f "$ echo "Main directory: $MAIN_DIR" mkdir -p "$MAIN_DIR"/tmp/ -readarray -t RELEASES < <( curl --silent --fail 'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1' | jq --raw-output '.results[].version' | head -n 5 | $SORT --version-sort ) || { echo "Failed to retrieve list of releases" >&2 ; exit 1 ; } +rm -rf "$MAIN_DIR"/tmp/generator/ +rm -rf "$MAIN_DIR"/tmp/generator.zip +wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/snapshots/org/jenkins-ci/update-center2/3.2.2-SNAPSHOT/update-center2-3.2.2-20200618.114038-2-bin.zip" +unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/ + +java -Dfile.encoding=UTF-8 -jar "$MAIN_DIR"/tmp/generator/update-center2-*.jar --dynamic-tier-list-file tmp/tiers.json +readarray -t WEEKLY_RELEASES < <( jq --raw-output '.weeklyCores[]' tmp/tiers.json ) || { echo "Failed to determine weekly tier list" >&2 ; exit 1 ; } +readarray -t STABLE_RELEASES < <( jq --raw-output '.stableCores[]' tmp/tiers.json ) || { echo "Failed to determine stable tier list" >&2 ; exit 1 ; } + +# Workaround for https://github.com/jenkinsci/docker/issues/954 -- still generate fixed tier update sites +readarray -t RELEASES < <( curl --silent --fail 'https://repo.jenkins-ci.org/api/search/versions?g=org.jenkins-ci.main&a=jenkins-core&repos=releases&v=?.*.1' | jq --raw-output '.results[].version' | head -n 5 | $SORT --version-sort ) || { echo "Failed to retrieve list of recent LTS releases" >&2 ; exit 1 ; } # prepare the www workspace for execution rm -rf "$WWW_ROOT_DIR" mkdir -p "$WWW_ROOT_DIR" # Generate htaccess file -"$( dirname "$0" )"/generate-htaccess.sh "${RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess" - -rm -rf "$MAIN_DIR"/tmp/generator/ -rm -rf "$MAIN_DIR"/tmp/generator.zip -wget --no-verbose -O "$MAIN_DIR"/tmp/generator.zip "https://repo.jenkins-ci.org/releases/org/jenkins-ci/update-center2/3.2.1/update-center2-3.2.1-bin.zip" -unzip -q "$MAIN_DIR"/tmp/generator.zip -d "$MAIN_DIR"/tmp/generator/ - +"$( dirname "$0" )"/generate-htaccess.sh "${WEEKLY_RELEASES[@]}" "${STABLE_RELEASES[@]}" > "$WWW_ROOT_DIR/.htaccess" # Reset arguments file echo "# one update site per line" > "$MAIN_DIR"/tmp/args.lst @@ -81,15 +85,23 @@ function sanity-check { fi } -# Generate several update sites for different segments so that plugins can +# Generate tiered update sites for different segments so that plugins can # aggressively update baseline requirements without stranding earlier users. # -# We use LTS as a boundary of different segments, to create -# a reasonable number of segments with reasonable sizes. Plugins -# tend to pick LTS baseline as the required version, so this works well. -# -# We generate tiered update sites for the five most recent LTS baselines, which -# means admins get compatible updates offered on releases up to about one year old. +# We generate tiered update sites for all core releases newer than +# about 13 months that are actually used as plugin dependencies. +# This supports updating Jenkins (core) once a year while getting offered compatible plugin updates. +for version in "${WEEKLY_RELEASES[@]}" ; do + # For mainline, advertising the latest core + generate --limit-plugin-core-dependency "$version" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/dynamic-$version/latest" --www-dir "$WWW_ROOT_DIR/dynamic-$version" +done + +for version in "${STABLE_RELEASES[@]}" ; do + # For LTS, advertising the latest LTS core + generate --limit-plugin-core-dependency "$version" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/dynamic-stable-$version/latest" --www-dir "$WWW_ROOT_DIR/dynamic-stable-$version" --only-stable-core +done + +# Workaround for https://github.com/jenkinsci/docker/issues/954 -- still generate fixed tier update sites for ltsv in "${RELEASES[@]}" ; do v="${ltsv/%.1/}" # For mainline up to $v, advertising the latest core @@ -99,7 +111,6 @@ for ltsv in "${RELEASES[@]}" ; do generate --limit-plugin-core-dependency "$v.999" --write-latest-core --latest-links-directory "$WWW_ROOT_DIR/stable-$v/latest" --www-dir "$WWW_ROOT_DIR/stable-$v" --only-stable-core done - # Experimental update center without version caps, including experimental releases. # This is not a part of the version-based redirection rules, admins need to manually configure it. # Generate this first, including --downloads-directory, as this includes all releases, experimental and otherwise. @@ -129,6 +140,16 @@ for ltsv in "${RELEASES[@]}" ; do lastLTS=$v done +for version in "${WEEKLY_RELEASES[@]}" ; do + sanity-check "$WWW_ROOT_DIR/dynamic-$version" + ln -sf ../updates "$WWW_ROOT_DIR/dynamic-$version/updates" +done + +for version in "${STABLE_RELEASES[@]}" ; do + sanity-check "$WWW_ROOT_DIR/dynamic-stable-$version" + ln -sf ../updates "$WWW_ROOT_DIR/dynamic-stable-$version/updates" +done + sanity-check "$WWW_ROOT_DIR/experimental" sanity-check "$WWW_ROOT_DIR/current" ln -sf ../updates "$WWW_ROOT_DIR/experimental/updates" diff --git a/site/static/readme.html b/site/static/readme.html index 22931eba4..b6f274f22 100644 --- a/site/static/readme.html +++ b/site/static/readme.html @@ -1,3 +1,9 @@ +

+ Important: + Do not rely on the existence or layout of subdirectories shown here. + To obtain versioned metadata for a specific Jenkins version, add a ?version= query parameter, for example `/update-center.json?version=2.234`. + Jenkins does this automatically, so the update site URL configured in Jenkins should always be just https://updates.jenkins.io/update-center.json. +

You can rsync these files via rsync -avz rsync://rsync.osuosl.org/jenkins/updates/ somewhere.

diff --git a/site/test/.gitignore b/site/test/.gitignore new file mode 100644 index 000000000..fb0308ca9 --- /dev/null +++ b/site/test/.gitignore @@ -0,0 +1 @@ +/htaccess.tmp diff --git a/site/test/Dockerfile b/site/test/Dockerfile new file mode 100644 index 000000000..78d390f24 --- /dev/null +++ b/site/test/Dockerfile @@ -0,0 +1,7 @@ +# Test image only containing the generated .htaccess file with redirects +FROM httpd:2.4 + +# It's the default file except we AllowOverride All, and enable mod_rewrite +COPY ./httpd.conf /usr/local/apache2/conf/httpd.conf +COPY ./htaccess.tmp /usr/local/apache2/htdocs/.htaccess +RUN chmod a+r /usr/local/apache2/htdocs/.htaccess diff --git a/site/test/httpd.conf b/site/test/httpd.conf new file mode 100644 index 000000000..9af3f5405 --- /dev/null +++ b/site/test/httpd.conf @@ -0,0 +1,551 @@ +# +# This is the main Apache HTTP server configuration file. It contains the +# configuration directives that give the server its instructions. +# See for detailed information. +# In particular, see +# +# for a discussion of each configuration directive. +# +# Do NOT simply read the instructions in here without understanding +# what they do. They're here only as hints or reminders. If you are unsure +# consult the online docs. You have been warned. +# +# Configuration and logfile names: If the filenames you specify for many +# of the server's control files begin with "/" (or "drive:/" for Win32), the +# server will use that explicit path. If the filenames do *not* begin +# with "/", the value of ServerRoot is prepended -- so "logs/access_log" +# with ServerRoot set to "/usr/local/apache2" will be interpreted by the +# server as "/usr/local/apache2/logs/access_log", whereas "/logs/access_log" +# will be interpreted as '/logs/access_log'. + +# +# ServerRoot: The top of the directory tree under which the server's +# configuration, error, and log files are kept. +# +# Do not add a slash at the end of the directory path. If you point +# ServerRoot at a non-local disk, be sure to specify a local disk on the +# Mutex directive, if file-based mutexes are used. If you wish to share the +# same ServerRoot for multiple httpd daemons, you will need to change at +# least PidFile. +# +ServerRoot "/usr/local/apache2" + +# +# Mutex: Allows you to set the mutex mechanism and mutex file directory +# for individual mutexes, or change the global defaults +# +# Uncomment and change the directory if mutexes are file-based and the default +# mutex file directory is not on a local disk or is not appropriate for some +# other reason. +# +# Mutex default:logs + +# +# Listen: Allows you to bind Apache to specific IP addresses and/or +# ports, instead of the default. See also the +# directive. +# +# Change this to Listen on specific IP addresses as shown below to +# prevent Apache from glomming onto all bound IP addresses. +# +#Listen 12.34.56.78:80 +Listen 80 + +# +# Dynamic Shared Object (DSO) Support +# +# To be able to use the functionality of a module which was built as a DSO you +# have to place corresponding `LoadModule' lines at this location so the +# directives contained in it are actually available _before_ they are used. +# Statically compiled modules (those listed by `httpd -l') do not need +# to be loaded here. +# +# Example: +# LoadModule foo_module modules/mod_foo.so +# +LoadModule mpm_event_module modules/mod_mpm_event.so +#LoadModule mpm_prefork_module modules/mod_mpm_prefork.so +#LoadModule mpm_worker_module modules/mod_mpm_worker.so +LoadModule authn_file_module modules/mod_authn_file.so +#LoadModule authn_dbm_module modules/mod_authn_dbm.so +#LoadModule authn_anon_module modules/mod_authn_anon.so +#LoadModule authn_dbd_module modules/mod_authn_dbd.so +#LoadModule authn_socache_module modules/mod_authn_socache.so +LoadModule authn_core_module modules/mod_authn_core.so +LoadModule authz_host_module modules/mod_authz_host.so +LoadModule authz_groupfile_module modules/mod_authz_groupfile.so +LoadModule authz_user_module modules/mod_authz_user.so +#LoadModule authz_dbm_module modules/mod_authz_dbm.so +#LoadModule authz_owner_module modules/mod_authz_owner.so +#LoadModule authz_dbd_module modules/mod_authz_dbd.so +LoadModule authz_core_module modules/mod_authz_core.so +#LoadModule authnz_ldap_module modules/mod_authnz_ldap.so +#LoadModule authnz_fcgi_module modules/mod_authnz_fcgi.so +LoadModule access_compat_module modules/mod_access_compat.so +LoadModule auth_basic_module modules/mod_auth_basic.so +#LoadModule auth_form_module modules/mod_auth_form.so +#LoadModule auth_digest_module modules/mod_auth_digest.so +#LoadModule allowmethods_module modules/mod_allowmethods.so +#LoadModule isapi_module modules/mod_isapi.so +#LoadModule file_cache_module modules/mod_file_cache.so +#LoadModule cache_module modules/mod_cache.so +#LoadModule cache_disk_module modules/mod_cache_disk.so +#LoadModule cache_socache_module modules/mod_cache_socache.so +#LoadModule socache_shmcb_module modules/mod_socache_shmcb.so +#LoadModule socache_dbm_module modules/mod_socache_dbm.so +#LoadModule socache_memcache_module modules/mod_socache_memcache.so +#LoadModule socache_redis_module modules/mod_socache_redis.so +#LoadModule watchdog_module modules/mod_watchdog.so +#LoadModule macro_module modules/mod_macro.so +#LoadModule dbd_module modules/mod_dbd.so +#LoadModule bucketeer_module modules/mod_bucketeer.so +#LoadModule dumpio_module modules/mod_dumpio.so +#LoadModule echo_module modules/mod_echo.so +#LoadModule example_hooks_module modules/mod_example_hooks.so +#LoadModule case_filter_module modules/mod_case_filter.so +#LoadModule case_filter_in_module modules/mod_case_filter_in.so +#LoadModule example_ipc_module modules/mod_example_ipc.so +#LoadModule buffer_module modules/mod_buffer.so +#LoadModule data_module modules/mod_data.so +#LoadModule ratelimit_module modules/mod_ratelimit.so +LoadModule reqtimeout_module modules/mod_reqtimeout.so +#LoadModule ext_filter_module modules/mod_ext_filter.so +#LoadModule request_module modules/mod_request.so +#LoadModule include_module modules/mod_include.so +LoadModule filter_module modules/mod_filter.so +#LoadModule reflector_module modules/mod_reflector.so +#LoadModule substitute_module modules/mod_substitute.so +#LoadModule sed_module modules/mod_sed.so +#LoadModule charset_lite_module modules/mod_charset_lite.so +#LoadModule deflate_module modules/mod_deflate.so +#LoadModule xml2enc_module modules/mod_xml2enc.so +#LoadModule proxy_html_module modules/mod_proxy_html.so +#LoadModule brotli_module modules/mod_brotli.so +LoadModule mime_module modules/mod_mime.so +#LoadModule ldap_module modules/mod_ldap.so +LoadModule log_config_module modules/mod_log_config.so +#LoadModule log_debug_module modules/mod_log_debug.so +#LoadModule log_forensic_module modules/mod_log_forensic.so +#LoadModule logio_module modules/mod_logio.so +#LoadModule lua_module modules/mod_lua.so +LoadModule env_module modules/mod_env.so +#LoadModule mime_magic_module modules/mod_mime_magic.so +#LoadModule cern_meta_module modules/mod_cern_meta.so +#LoadModule expires_module modules/mod_expires.so +LoadModule headers_module modules/mod_headers.so +#LoadModule ident_module modules/mod_ident.so +#LoadModule usertrack_module modules/mod_usertrack.so +#LoadModule unique_id_module modules/mod_unique_id.so +LoadModule setenvif_module modules/mod_setenvif.so +LoadModule version_module modules/mod_version.so +#LoadModule remoteip_module modules/mod_remoteip.so +#LoadModule proxy_module modules/mod_proxy.so +#LoadModule proxy_connect_module modules/mod_proxy_connect.so +#LoadModule proxy_ftp_module modules/mod_proxy_ftp.so +#LoadModule proxy_http_module modules/mod_proxy_http.so +#LoadModule proxy_fcgi_module modules/mod_proxy_fcgi.so +#LoadModule proxy_scgi_module modules/mod_proxy_scgi.so +#LoadModule proxy_uwsgi_module modules/mod_proxy_uwsgi.so +#LoadModule proxy_fdpass_module modules/mod_proxy_fdpass.so +#LoadModule proxy_wstunnel_module modules/mod_proxy_wstunnel.so +#LoadModule proxy_ajp_module modules/mod_proxy_ajp.so +#LoadModule proxy_balancer_module modules/mod_proxy_balancer.so +#LoadModule proxy_express_module modules/mod_proxy_express.so +#LoadModule proxy_hcheck_module modules/mod_proxy_hcheck.so +#LoadModule session_module modules/mod_session.so +#LoadModule session_cookie_module modules/mod_session_cookie.so +#LoadModule session_crypto_module modules/mod_session_crypto.so +#LoadModule session_dbd_module modules/mod_session_dbd.so +#LoadModule slotmem_shm_module modules/mod_slotmem_shm.so +#LoadModule slotmem_plain_module modules/mod_slotmem_plain.so +#LoadModule ssl_module modules/mod_ssl.so +#LoadModule optional_hook_export_module modules/mod_optional_hook_export.so +#LoadModule optional_hook_import_module modules/mod_optional_hook_import.so +#LoadModule optional_fn_import_module modules/mod_optional_fn_import.so +#LoadModule optional_fn_export_module modules/mod_optional_fn_export.so +#LoadModule dialup_module modules/mod_dialup.so +#LoadModule http2_module modules/mod_http2.so +#LoadModule proxy_http2_module modules/mod_proxy_http2.so +#LoadModule md_module modules/mod_md.so +#LoadModule lbmethod_byrequests_module modules/mod_lbmethod_byrequests.so +#LoadModule lbmethod_bytraffic_module modules/mod_lbmethod_bytraffic.so +#LoadModule lbmethod_bybusyness_module modules/mod_lbmethod_bybusyness.so +#LoadModule lbmethod_heartbeat_module modules/mod_lbmethod_heartbeat.so +LoadModule unixd_module modules/mod_unixd.so +#LoadModule heartbeat_module modules/mod_heartbeat.so +#LoadModule heartmonitor_module modules/mod_heartmonitor.so +#LoadModule dav_module modules/mod_dav.so +LoadModule status_module modules/mod_status.so +LoadModule autoindex_module modules/mod_autoindex.so +#LoadModule asis_module modules/mod_asis.so +#LoadModule info_module modules/mod_info.so +#LoadModule suexec_module modules/mod_suexec.so + + #LoadModule cgid_module modules/mod_cgid.so + + + #LoadModule cgi_module modules/mod_cgi.so + +#LoadModule dav_fs_module modules/mod_dav_fs.so +#LoadModule dav_lock_module modules/mod_dav_lock.so +#LoadModule vhost_alias_module modules/mod_vhost_alias.so +#LoadModule negotiation_module modules/mod_negotiation.so +LoadModule dir_module modules/mod_dir.so +#LoadModule imagemap_module modules/mod_imagemap.so +#LoadModule actions_module modules/mod_actions.so +#LoadModule speling_module modules/mod_speling.so +#LoadModule userdir_module modules/mod_userdir.so +LoadModule alias_module modules/mod_alias.so +LoadModule rewrite_module modules/mod_rewrite.so + + +# +# If you wish httpd to run as a different user or group, you must run +# httpd as root initially and it will switch. +# +# User/Group: The name (or #number) of the user/group to run httpd as. +# It is usually good practice to create a dedicated user and group for +# running httpd, as with most system services. +# +User daemon +Group daemon + + + +# 'Main' server configuration +# +# The directives in this section set up the values used by the 'main' +# server, which responds to any requests that aren't handled by a +# definition. These values also provide defaults for +# any containers you may define later in the file. +# +# All of these directives may appear inside containers, +# in which case these default settings will be overridden for the +# virtual host being defined. +# + +# +# ServerAdmin: Your address, where problems with the server should be +# e-mailed. This address appears on some server-generated pages, such +# as error documents. e.g. admin@your-domain.com +# +ServerAdmin you@example.com + +# +# ServerName gives the name and port that the server uses to identify itself. +# This can often be determined automatically, but we recommend you specify +# it explicitly to prevent problems during startup. +# +# If your host doesn't have a registered DNS name, enter its IP address here. +# +#ServerName www.example.com:80 + +# +# Deny access to the entirety of your server's filesystem. You must +# explicitly permit access to web content directories in other +# blocks below. +# + + AllowOverride none + Require all denied + + +# +# Note that from this point forward you must specifically allow +# particular features to be enabled - so if something's not working as +# you might expect, make sure that you have specifically enabled it +# below. +# + +# +# DocumentRoot: The directory out of which you will serve your +# documents. By default, all requests are taken from this directory, but +# symbolic links and aliases may be used to point to other locations. +# +DocumentRoot "/usr/local/apache2/htdocs" + + # + # Possible values for the Options directive are "None", "All", + # or any combination of: + # Indexes Includes FollowSymLinks SymLinksifOwnerMatch ExecCGI MultiViews + # + # Note that "MultiViews" must be named *explicitly* --- "Options All" + # doesn't give it to you. + # + # The Options directive is both complicated and important. Please see + # http://httpd.apache.org/docs/2.4/mod/core.html#options + # for more information. + # + Options Indexes FollowSymLinks + + # + # AllowOverride controls what directives may be placed in .htaccess files. + # It can be "All", "None", or any combination of the keywords: + # AllowOverride FileInfo AuthConfig Limit + # + AllowOverride All + + # + # Controls who can get stuff from this server. + # + Require all granted + + +# +# DirectoryIndex: sets the file that Apache will serve if a directory +# is requested. +# + + DirectoryIndex index.html + + +# +# The following lines prevent .htaccess and .htpasswd files from being +# viewed by Web clients. +# + + Require all denied + + +# +# ErrorLog: The location of the error log file. +# If you do not specify an ErrorLog directive within a +# container, error messages relating to that virtual host will be +# logged here. If you *do* define an error logfile for a +# container, that host's errors will be logged there and not here. +# +ErrorLog /proc/self/fd/2 + +# +# LogLevel: Control the number of messages logged to the error_log. +# Possible values include: debug, info, notice, warn, error, crit, +# alert, emerg. +# +LogLevel warn + + + # + # The following directives define some format nicknames for use with + # a CustomLog directive (see below). + # + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + # You need to enable mod_logio.c to use %I and %O + LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + # + # The location and format of the access logfile (Common Logfile Format). + # If you do not define any access logfiles within a + # container, they will be logged here. Contrariwise, if you *do* + # define per- access logfiles, transactions will be + # logged therein and *not* in this file. + # + CustomLog /proc/self/fd/1 common + + # + # If you prefer a logfile with access, agent, and referer information + # (Combined Logfile Format) you can use the following directive. + # + #CustomLog "logs/access_log" combined + + + + # + # Redirect: Allows you to tell clients about documents that used to + # exist in your server's namespace, but do not anymore. The client + # will make a new request for the document at its new location. + # Example: + # Redirect permanent /foo http://www.example.com/bar + + # + # Alias: Maps web paths into filesystem paths and is used to + # access content that does not live under the DocumentRoot. + # Example: + # Alias /webpath /full/filesystem/path + # + # If you include a trailing / on /webpath then the server will + # require it to be present in the URL. You will also likely + # need to provide a section to allow access to + # the filesystem path. + + # + # ScriptAlias: This controls which directories contain server scripts. + # ScriptAliases are essentially the same as Aliases, except that + # documents in the target directory are treated as applications and + # run by the server when requested rather than as documents sent to the + # client. The same rules about trailing "/" apply to ScriptAlias + # directives as to Alias. + # + ScriptAlias /cgi-bin/ "/usr/local/apache2/cgi-bin/" + + + + + # + # ScriptSock: On threaded servers, designate the path to the UNIX + # socket used to communicate with the CGI daemon of mod_cgid. + # + #Scriptsock cgisock + + +# +# "/usr/local/apache2/cgi-bin" should be changed to whatever your ScriptAliased +# CGI directory exists, if you have that configured. +# + + AllowOverride None + Options None + Require all granted + + + + # + # Avoid passing HTTP_PROXY environment to CGI's on this or any proxied + # backend servers which have lingering "httpoxy" defects. + # 'Proxy' request header is undefined by the IETF, not listed by IANA + # + RequestHeader unset Proxy early + + + + # + # TypesConfig points to the file containing the list of mappings from + # filename extension to MIME-type. + # + TypesConfig conf/mime.types + + # + # AddType allows you to add to or override the MIME configuration + # file specified in TypesConfig for specific file types. + # + #AddType application/x-gzip .tgz + # + # AddEncoding allows you to have certain browsers uncompress + # information on the fly. Note: Not all browsers support this. + # + #AddEncoding x-compress .Z + #AddEncoding x-gzip .gz .tgz + # + # If the AddEncoding directives above are commented-out, then you + # probably should define those extensions to indicate media types: + # + AddType application/x-compress .Z + AddType application/x-gzip .gz .tgz + + # + # AddHandler allows you to map certain file extensions to "handlers": + # actions unrelated to filetype. These can be either built into the server + # or added with the Action directive (see below) + # + # To use CGI scripts outside of ScriptAliased directories: + # (You will also need to add "ExecCGI" to the "Options" directive.) + # + #AddHandler cgi-script .cgi + + # For type maps (negotiated resources): + #AddHandler type-map var + + # + # Filters allow you to process content before it is sent to the client. + # + # To parse .shtml files for server-side includes (SSI): + # (You will also need to add "Includes" to the "Options" directive.) + # + #AddType text/html .shtml + #AddOutputFilter INCLUDES .shtml + + +# +# The mod_mime_magic module allows the server to use various hints from the +# contents of the file itself to determine its type. The MIMEMagicFile +# directive tells the module where the hint definitions are located. +# +#MIMEMagicFile conf/magic + +# +# Customizable error responses come in three flavors: +# 1) plain text 2) local redirects 3) external redirects +# +# Some examples: +#ErrorDocument 500 "The server made a boo boo." +#ErrorDocument 404 /missing.html +#ErrorDocument 404 "/cgi-bin/missing_handler.pl" +#ErrorDocument 402 http://www.example.com/subscription_info.html +# + +# +# MaxRanges: Maximum number of Ranges in a request before +# returning the entire resource, or one of the special +# values 'default', 'none' or 'unlimited'. +# Default setting is to accept 200 Ranges. +#MaxRanges unlimited + +# +# EnableMMAP and EnableSendfile: On systems that support it, +# memory-mapping or the sendfile syscall may be used to deliver +# files. This usually improves server performance, but must +# be turned off when serving from networked-mounted +# filesystems or if support for these functions is otherwise +# broken on your system. +# Defaults: EnableMMAP On, EnableSendfile Off +# +#EnableMMAP off +#EnableSendfile on + +# Supplemental configuration +# +# The configuration files in the conf/extra/ directory can be +# included to add extra features or to modify the default configuration of +# the server, or you may simply copy their contents here and change as +# necessary. + +# Server-pool management (MPM specific) +#Include conf/extra/httpd-mpm.conf + +# Multi-language error messages +#Include conf/extra/httpd-multilang-errordoc.conf + +# Fancy directory listings +#Include conf/extra/httpd-autoindex.conf + +# Language settings +#Include conf/extra/httpd-languages.conf + +# User home directories +#Include conf/extra/httpd-userdir.conf + +# Real-time info on requests and configuration +#Include conf/extra/httpd-info.conf + +# Virtual hosts +#Include conf/extra/httpd-vhosts.conf + +# Local access to the Apache HTTP Server Manual +#Include conf/extra/httpd-manual.conf + +# Distributed authoring and versioning (WebDAV) +#Include conf/extra/httpd-dav.conf + +# Various default settings +#Include conf/extra/httpd-default.conf + +# Configure mod_proxy_html to understand HTML4/XHTML1 + +Include conf/extra/proxy-html.conf + + +# Secure (SSL/TLS) connections +#Include conf/extra/httpd-ssl.conf +# +# Note: The following must must be present to support +# starting without SSL on platforms with no /dev/random equivalent +# but a statically compiled-in mod_ssl. +# + +SSLRandomSeed startup builtin +SSLRandomSeed connect builtin + + diff --git a/site/test/redirects.sh b/site/test/redirects.sh deleted file mode 100755 index c814983ae..000000000 --- a/site/test/redirects.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -# Simple script to manually test actual output is sane -- This is not automatically run as part of the build (since it needs a full deployment first) -# Ideally we'd set up a local test server with the output, but for now, this will help us quickly figure out if things broke. - -CURRENT_WEEKLY=$(curl -sL https://updates.jenkins.io/latestCore.txt) -RECENT_WEEKLY=$(echo ${CURRENT_WEEKLY} | cut -d. -f1).$(($(echo ${CURRENT_WEEKLY} | cut -d. -f2)-1)) -NEWEST_LTS_BASELINE=$(curl -sL https://updates.jenkins.io/stable/latestCore.txt) -NEWEST_LTS_BASELINE=${NEWEST_LTS_BASELINE%.*} -OLDEST_SUPPORTED_LTS_BASELINE=$(curl -sL https://updates.jenkins.io | grep -E "stable-([0-9]+\.[0-9]+).*/\1/' | sort -V | head -n1) - -set -o pipefail -set -o errexit -set -o nounset - -function test_redirect () { - local REQUEST_URL="$1" - local DESTINATION="$2" - echo "Requesting $REQUEST_URL (-> $DESTINATION)" - REDIRECT=$( curl -Ii "$REQUEST_URL" 2>/dev/null | fgrep 'Location:' | cut -d' ' -f2 | tr -d '[:space:]' ) || { echo "Failed to curl $REQUEST_URL" >&2 ; } - if [ $REDIRECT != $DESTINATION ] ; then - echo "Expected $DESTINATION but got $REDIRECT for $REQUEST_URL" - fi -} - -test_redirect "https://updates.jenkins.io/update-center.json" "https://updates.jenkins.io/current/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json.html" "https://updates.jenkins.io/current/update-center.json.html" -test_redirect "https://updates.jenkins.io/update-center.actual.json" "https://updates.jenkins.io/current/update-center.actual.json" -test_redirect "https://updates.jenkins.io/release-history.json" "https://updates.jenkins.io/current/release-history.json" -test_redirect "https://updates.jenkins.io/plugin-versions.json" "https://updates.jenkins.io/current/plugin-versions.json" -test_redirect "https://updates.jenkins.io/plugin-documentation-urls.json" "https://updates.jenkins.io/current/plugin-documentation-urls.json" -test_redirect "https://updates.jenkins.io/latestCore.txt" "https://updates.jenkins.io/current/latestCore.txt" - - -test_redirect "https://updates.jenkins.io/stable/update-center.json" "https://updates.jenkins.io/stable-$NEWEST_LTS_BASELINE/update-center.json" - -test_redirect "https://updates.jenkins.io/update-center.json?version=$CURRENT_WEEKLY" "https://updates.jenkins.io/current/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=$RECENT_WEEKLY" "https://updates.jenkins.io/current/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=$NEWEST_LTS_BASELINE" "https://updates.jenkins.io/$NEWEST_LTS_BASELINE/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=$NEWEST_LTS_BASELINE.1" "https://updates.jenkins.io/stable-$NEWEST_LTS_BASELINE/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=2.55" "https://updates.jenkins.io/$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=2.6" "https://updates.jenkins.io/$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" - -test_redirect "https://updates.jenkins.io/update-center.actual.json?version=$NEWEST_LTS_BASELINE" "https://updates.jenkins.io/$NEWEST_LTS_BASELINE/update-center.actual.json" -test_redirect "https://updates.jenkins.io/update-center.actual.json?version=$NEWEST_LTS_BASELINE.1" "https://updates.jenkins.io/stable-$NEWEST_LTS_BASELINE/update-center.actual.json" -test_redirect "https://updates.jenkins.io/plugin-documentation-urls.json?version=$NEWEST_LTS_BASELINE.1" "https://updates.jenkins.io/stable-$NEWEST_LTS_BASELINE/plugin-documentation-urls.json" -test_redirect "https://updates.jenkins.io/latestCore.txt?version=$NEWEST_LTS_BASELINE.1" "https://updates.jenkins.io/stable-$NEWEST_LTS_BASELINE/latestCore.txt" - -# Jenkins 1.x gets the newest update sites, we don't care enough to have a more appropriate redirect for them -#test_redirect "https://updates.jenkins.io/update-center.json?version=1.650" "https://updates.jenkins.io/$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" -#test_redirect "https://updates.jenkins.io/update-center.json?version=1.580" "https://updates.jenkins.io/$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" - -test_redirect "https://updates.jenkins.io/update-center.json?version=1.580.1" "https://updates.jenkins.io/stable-$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" -test_redirect "https://updates.jenkins.io/update-center.json?version=2.46.1" "https://updates.jenkins.io/stable-$OLDEST_SUPPORTED_LTS_BASELINE/update-center.json" - -# Where to redirect a slightly older non-baseline weekly release: -# This would probably be ideal: Drop down if older than newest LTS baseline, this instance isn't getting updates weekly -#test_redirect "https://updates.jenkins.io/update-center.json?version=2.120" "https://updates.jenkins.io/2.107/update-center.json" -# The following describes the current behavior. Second argument is a recent LTS baseline (neither oldest nor newest supported); first argument is a weekly release a bit earlier than that. -# While still true when it's the oldest, then this is the expected result. -test_redirect "https://updates.jenkins.io/update-center.json?version=2.200" "https://updates.jenkins.io/2.204/update-center.json" diff --git a/site/test/test.sh b/site/test/test.sh new file mode 100755 index 000000000..e1dabd481 --- /dev/null +++ b/site/test/test.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail + +SIMPLE_SCRIPT_DIR="$( dirname "$0" )" +SCRIPT_DIR="$( readlink -f "${SIMPLE_SCRIPT_DIR}" 2>/dev/null || greadlink -f "${SIMPLE_SCRIPT_DIR}" )" || { echo "Failed to determine script directory using (g)readlink -f" >&2 ; exit 1 ; } + +"${SCRIPT_DIR}"/../generate-htaccess.sh \ + 2.172 2.173 2.176 2.177 2.181 2.184 2.185 2.191 2.195 2.199 2.204 2.205 2.212 2.217 2.222 2.223 \ + 2.164.2 2.164.3 2.176.1 2.176.2 2.176.3 2.176.4 2.190.1 2.190.3 2.204.1 2.204.2 2.204.4 2.204.6 2.222.1 \ + > "${SCRIPT_DIR}/htaccess.tmp" + +docker build -t update-center2-test "${SCRIPT_DIR}" || { echo "Failed to build Docker image" >&2 ; exit 1 ; } + +trap 'cleanup' 0 +function cleanup { + docker stop update-center2-test +} +docker run --rm -dit --name update-center2-test -p 8080:80 update-center2-test || { echo "Failed to start Docker container" >&2 ; exit 1 ; } + +TEST_BASE_URL="http://localhost:8080" + +function test_redirect () { + local REQUEST_URL="$1" + local DESTINATION="$2" + echo "Requesting $REQUEST_URL (-> $DESTINATION)" + REDIRECT=$( curl -Ii "$REQUEST_URL" 2>/dev/null | fgrep 'Location:' | cut -d' ' -f2 | tr -d '[:space:]' ) || { echo "Failed to curl $REQUEST_URL" >&2 ; } + if [ "$REDIRECT" != "$DESTINATION" ] ; then + echo "Expected $DESTINATION but got $REDIRECT for $REQUEST_URL" + fi +} + +test_redirect "$TEST_BASE_URL/update-center.json" "$TEST_BASE_URL/current/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json.html" "$TEST_BASE_URL/current/update-center.json.html" + +# Accessed by https://github.com/jenkins-infra/jenkins.io/blob/3892ea2ad4b4a67e1f8aebbfab261ae88628c176/scripts/fetch-external-resources#L48 +test_redirect "$TEST_BASE_URL/update-center.actual.json" "$TEST_BASE_URL/current/update-center.actual.json" + +# Accessed by https://github.com/jenkins-infra/jenkins.io/blob/a9d12fe7234199ee8c4426f6eca98b71de8dffbd/Makefile#L46 +# Target accessed by https://github.com/jenkins-infra/plugin-site-api/blob/bd2a5fe337ddf9526666ef5a7ad6a3f5bb388c6f/src/main/java/io/jenkins/plugins/generate/parsers/WikiPluginDataParser.java#L30 +test_redirect "$TEST_BASE_URL/release-history.json" "$TEST_BASE_URL/current/release-history.json" + +# Target accessed by https://github.com/jenkinsci/plugin-installation-manager-tool/blob/fbdbd6b8e8e291db28fadc4ad4b5ec9795bd3c37/plugin-management-library/src/main/java/io/jenkins/tools/pluginmanager/config/Settings.java#L16 +test_redirect "$TEST_BASE_URL/plugin-versions.json" "$TEST_BASE_URL/current/plugin-versions.json" +test_redirect "$TEST_BASE_URL/plugin-documentation-urls.json" "$TEST_BASE_URL/current/plugin-documentation-urls.json" + +# Accessed by https://github.com/jenkins-infra/jenkins.io/blob/3892ea2ad4b4a67e1f8aebbfab261ae88628c176/scripts/fetch-external-resources#L18 +test_redirect "$TEST_BASE_URL/latestCore.txt" "$TEST_BASE_URL/current/latestCore.txt" + + +test_redirect "$TEST_BASE_URL/stable/update-center.json" "$TEST_BASE_URL/dynamic-stable-2.222.1/update-center.json" + +# Accessed by https://github.com/jenkins-infra/jenkins.io/blob/3892ea2ad4b4a67e1f8aebbfab261ae88628c176/scripts/fetch-external-resources#L24 +test_redirect "$TEST_BASE_URL/stable/latestCore.txt" "$TEST_BASE_URL/dynamic-stable-2.222.1/latestCore.txt" + +test_redirect "$TEST_BASE_URL/update-center.json?version=2.236" "$TEST_BASE_URL/dynamic-2.223/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=2.230" "$TEST_BASE_URL/dynamic-2.223/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=2.222" "$TEST_BASE_URL/dynamic-2.217/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=2.222.1" "$TEST_BASE_URL/dynamic-stable-2.222.1/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=2.55" "$TEST_BASE_URL/dynamic-2.172/update-center.json" # TODO Fix +test_redirect "$TEST_BASE_URL/update-center.json?version=2.6" "$TEST_BASE_URL/dynamic-2.172/update-center.json" # TODO Fix + +test_redirect "$TEST_BASE_URL/update-center.actual.json?version=2.222" "$TEST_BASE_URL/dynamic-2.217/update-center.actual.json" +test_redirect "$TEST_BASE_URL/update-center.actual.json?version=2.222.1" "$TEST_BASE_URL/dynamic-stable-2.222.1/update-center.actual.json" + +# No more redirects to tiers +test_redirect "$TEST_BASE_URL/plugin-documentation-urls.json?version=2.222.1" "$TEST_BASE_URL/current/plugin-documentation-urls.json" +test_redirect "$TEST_BASE_URL/latestCore.txt?version=2.222.1" "$TEST_BASE_URL/current/latestCore.txt" + +# Jenkins 1.x gets the oldest update sites +test_redirect "$TEST_BASE_URL/update-center.json?version=1.650" "$TEST_BASE_URL/dynamic-2.172/update-center.json" # TODO Fix +test_redirect "$TEST_BASE_URL/update-center.json?version=1.580" "$TEST_BASE_URL/dynamic-2.172/update-center.json" # TODO Fix +test_redirect "$TEST_BASE_URL/update-center.json?version=1.580.1" "$TEST_BASE_URL/dynamic-stable-2.164.2/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=2.46.1" "$TEST_BASE_URL/dynamic-stable-2.164.2/update-center.json" + +# This would probably be ideal: Drop down if older than newest LTS baseline, this instance isn't getting updates weekly +test_redirect "$TEST_BASE_URL/update-center.json?version=2.200" "$TEST_BASE_URL/dynamic-2.199/update-center.json" + +# Future major releases go to the most recent update sites: +test_redirect "$TEST_BASE_URL/update-center.json?version=3.0" "$TEST_BASE_URL/dynamic-2.223/update-center.json" +test_redirect "$TEST_BASE_URL/update-center.json?version=3.0.1" "$TEST_BASE_URL/dynamic-stable-2.222.1/update-center.json" diff --git a/src/main/java/io/jenkins/update_center/Main.java b/src/main/java/io/jenkins/update_center/Main.java index d4d8393ee..fe0c7f31c 100644 --- a/src/main/java/io/jenkins/update_center/Main.java +++ b/src/main/java/io/jenkins/update_center/Main.java @@ -27,6 +27,7 @@ import hudson.util.VersionNumber; import io.jenkins.lib.support_log_formatter.SupportLogFormatter; import io.jenkins.update_center.args4j.LevelOptionHandler; +import io.jenkins.update_center.json.TieredUpdateSitesGenerator; import io.jenkins.update_center.json.PluginDocumentationUrlsRoot; import io.jenkins.update_center.wrappers.AlphaBetaOnlyRepository; import io.jenkins.update_center.wrappers.StableWarMavenRepository; @@ -105,6 +106,9 @@ public class Main { /* Configure what kinds of output to generate */ + @Option(name = "--dynamic-tier-list-file", usage = "Generate tier list JSON file at the specified path. If this option is set, we skip generating all other output.") + @CheckForNull public File tierListFile; + @Option(name = "--www-dir", usage = "Generate simple output files, JSON(ish) and others, into this directory") @CheckForNull public File www; @@ -228,6 +232,11 @@ public void run() throws Exception { MavenRepository repo = createRepository(); initializeLatestPluginVersions(skipLatestPluginRelease); + if (tierListFile != null) { + new TieredUpdateSitesGenerator().withRepository(repo).write(tierListFile, prettyPrint); + return; + } + metadataWriter.writeMetadataFiles(repo, www); if (!skipUpdateCenter) { @@ -263,7 +272,7 @@ private String updateCenterPostMessageHtml(String updateCenterJson) { private static void writeToFile(String string, final File file) throws IOException { File parentFile = file.getParentFile(); - if (!parentFile.isDirectory() && !parentFile.mkdirs()) { + if (parentFile != null && !parentFile.isDirectory() && !parentFile.mkdirs()) { throw new IOException("Failed to create parent directory " + parentFile); } PrintWriter rhpw = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8)); diff --git a/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java b/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java new file mode 100644 index 000000000..f0e77f1a0 --- /dev/null +++ b/src/main/java/io/jenkins/update_center/json/TieredUpdateSitesGenerator.java @@ -0,0 +1,136 @@ +/* + * The MIT License + * + * Copyright (c) 2020, Daniel Beck + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package io.jenkins.update_center.json; + +import com.alibaba.fastjson.annotation.JSONField; +import hudson.util.VersionNumber; +import io.jenkins.update_center.HPI; +import io.jenkins.update_center.JenkinsWar; +import io.jenkins.update_center.MavenRepository; + +import java.io.IOException; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +public class TieredUpdateSitesGenerator extends WithoutSignature { + + private MavenRepository repository; + + @JSONField + public List weeklyCores; + + @JSONField + public List stableCores; + + public TieredUpdateSitesGenerator withRepository(MavenRepository repository) throws IOException { + this.repository = repository; + update(); + return this; + } + + private static boolean isStableVersion(VersionNumber version) { + return version.getDigitAt(2) != -1; + } + + private static VersionNumber nextWeeklyReleaseAfterStableBaseline(VersionNumber version) { + if (!version.toString().matches("[0-9][.][0-9]+[.][1-9]")) { + throw new IllegalArgumentException("Unexpected LTS version: " + version.toString()); + } + return new VersionNumber(version.getDigitAt(0) + "." + (version.getDigitAt(1) + 1)); + } + + private static boolean isReleaseRecentEnough(JenkinsWar war) { + Objects.requireNonNull(war, "war"); + return war.getTimestampAsDate().toInstant().isAfter(Instant.now().minus(CORE_AGE_DAYS, ChronoUnit.DAYS)); + } + + public void update() throws IOException { + Collection allPluginReleases = this.repository.listJenkinsPlugins().stream() + .map(plugin -> plugin.getArtifacts().values()) + .reduce(new HashSet<>(), (acc, els) -> { acc.addAll(els); return acc; }); + + final List coreDependencyVersions = allPluginReleases.stream().map(v -> { + try { + return v.getRequiredJenkinsVersion(); + } catch (IOException e) { + LOGGER.log(Level.WARNING, "Failed to determine required Jenkins version for " + v.getGavId()); + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toSet()).stream().map(VersionNumber::new).sorted(Comparator.reverseOrder()).collect(Collectors.toList()); + + final TreeMap allJenkinsWarsByVersionNumber = this.repository.getJenkinsWarsByVersionNumber(); + final Set weeklyCores = new HashSet<>(); + final Set stableCores = new HashSet<>(); + + boolean stableDone = false; + boolean weeklyDone = false; + + for (VersionNumber dependencyVersion : coreDependencyVersions) { + final JenkinsWar war = allJenkinsWarsByVersionNumber.get(dependencyVersion); + if (war == null) { + LOGGER.log(Level.INFO, "Did not find declared core dependency version among all core releases: " + dependencyVersion.toString()); + continue; + } + final boolean releaseRecentEnough = isReleaseRecentEnough(war); + if (isStableVersion(dependencyVersion)) { + if (!stableDone) { + if (!releaseRecentEnough) { + stableDone = true; + } + stableCores.add(dependencyVersion); + if (!weeklyDone) { + weeklyCores.add(nextWeeklyReleaseAfterStableBaseline(dependencyVersion)); + } + } + } else { + if (!weeklyDone) { + if (!releaseRecentEnough) { + weeklyDone = true; + } + weeklyCores.add(dependencyVersion); + } + } + if (stableDone && weeklyDone) { + break; + } + } + + this.stableCores = stableCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList()); + this.weeklyCores = weeklyCores.stream().map(VersionNumber::toString).sorted().collect(Collectors.toList()); + } + + public static final Logger LOGGER = Logger.getLogger(TieredUpdateSitesGenerator.class.getName()); + + private static final int CORE_AGE_DAYS = 400; +}