Improve retries in the installation scripts (#308)

.github/workflows/ci.yml

@@ -195,6 +195,12 @@ jobs:
  #
  # At least this checks that our installation script works as expected.
  - uses: ./
+ env:
+ # Given the default retry sequence of 1, 2, 4, 8, 16, 32, 60, 60, 60, ... 60 (seconds),
+ # this number of retries will give us approximately 30 minutes of total retry time.
+ # This is necessary to make CI wait for the release artifacts to be available on Github
+ # when a new version is released.
+ MARKER_NET_RETRY_COUNT: 35
  with:
  install-only: true
 

docs/book/src/usage/ci.md

@@ -38,6 +38,16 @@ All inputs are optional, they only allow tweaking the default behavior.
 |----------------|---------------------------------------------------------------|-----------|---------|
 | `install-only` | Only install Marker but don't run the `cargo marker` command. | `boolean` | `false` |
 
+
+### Environment variables
+
+| Name | Description | Type | Default |
+|------------------------------|--------------------------------------------------------------------------|-----------|---------|
+| `MARKER_NET_RETRY_COUNT` | Max number of retries for downloads. This also sets `RUSTUP_MAX_RETRIES` | `integer` | `5` |
+| `MARKER_NET_RETRY_MAX_DELAY` | Max delay between subsequent retries for downloads in seconds | `integer` | `60` |
+
+These environment variables configure the behavior of the installation script and they may be used if you run that script directly as well e.g. on [other CI systems](#other-ci-systems).
+
 ### Example workflows
 
 These example workflows will use the lint crates specified in the `Cargo.toml` file by default. Refer to the [*Lint Crate Declaration*](./lint-crate-declaration.md) section for more information.
@@ -115,7 +125,7 @@ curl \
  --fail \
  --show-error \
  --retry 5 \
- --retry-connrefused \
+ --retry-all-errors \
  https://raw.githubusercontent.com/rust-marker/marker/v0.3/scripts/release/install.sh \
  | bash
 ```
@@ -128,14 +138,17 @@ curl.exe `
  --fail `
  --show-error `
  --retry 5 `
- --retry-connrefused `
+ --retry-all-errors `
  https://raw.githubusercontent.com/rust-marker/marker/v0.3/scripts/release/install.ps1 `
  | powershell -command -
 ```
 
 <!-- endregion replace marker version stable -->
 
+Both of these scripts are configurable. See the [environment variables](#environment-variables) for details on what's available.
+
 The available version git tags that you may use in the URL are described in the [git tags](#git-tags) paragraph of the Github Action.
 
+[`RUSTUP_MAX_RETRIES`]: https://github.com/rust-lang/rustup/blob/5af4bc4a0d4bc69ea9091a7935fb3783c5fb508e/doc/dev-guide/src/tips-and-tricks.md#rustup_max_retries
 [new issue]: https://gitHub.com/rust-marker/marker/issues/new/choose
 [OS images supported by managed GitHub Actions runners]: https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources

scripts/download/lib.sh

@@ -75,14 +75,17 @@ function try_download_and_decompress {
  rm $archive
 }
 
+# Be careful to use this only for HTTP GET requests! This script does aggressive
+# retries, so if you use it for a non-readonly HTTP method that doesn't use any
+# idempotency token validation mechanism you might end up with duplicate modifications.
 function curl_with_retry {
  with_log curl \
  --location \
  --silent \
  --show-error \
  --fail \
  --retry 5 \
- --retry-connrefused \
+ --retry-all-errors \
  "$@"
 }
 

scripts/release/install.ps1

@@ -40,9 +40,31 @@
 # | Command curl failed with exit code 22
 # ```
 
-
 $ErrorActionPreference = "Stop"
 
+Write-Output "PowerShell version: $($PSVersionTable.PSVersion)"
+
+# Configure the retries for downloading the release assets
+if ($env:MARKER_NET_RETRY_COUNT -eq $null) {
+ $env:MARKER_NET_RETRY_COUNT = '5'
+}
+
+if ($env:MARKER_NET_RETRY_MAX_DELAY -eq $null) {
+ $env:MARKER_NET_RETRY_MAX_DELAY = '60'
+}
+
+if ($env:RUSTUP_MAX_RETRIES -eq $null) {
+ $env:RUSTUP_MAX_RETRIES = $env:MARKER_NET_RETRY_COUNT
+}
+
+Write-Output @"
+Using config env vars (override these if needed):
+ MARKER_NET_RETRY_COUNT=$env:MARKER_NET_RETRY_COUNT
+ MARKER_NET_RETRY_MAX_DELAY=$env:MARKER_NET_RETRY_MAX_DELAY
+ RUSTUP_MAX_RETRIES=$env:RUSTUP_MAX_RETRIES
+"@
+
+
 # This script isn't meant to be run from `master`, but if it is, then
 # it will install the latest version be it a stable version or a pre-release.
 # region replace marker version unstable
@@ -151,8 +173,6 @@ function Check-Sha256Sum {
  Write-Error "No checksum found for $file"
 }
 
-Write-Output "PowerShell version: $($PSVersionTable.PSVersion)"
-
 # This script can run on unix too if you have PowerShell installed there.
 # The only difference is that on Windows's old PowerShell 5 there is an alias
 # `curl` for `Invoke-WebRequest`, and if you want to use the real curl, then
@@ -194,8 +214,9 @@ try {
  --silent `
  --fail `
  --show-error `
- --retry 5 `
- --retry-connrefused `
+ --retry $env:MARKER_NET_RETRY_COUNT `
+ --retry-max-time $env:MARKER_NET_RETRY_MAX_DELAY `
+ --retry-all-errors `
  --remote-name `
  --output-dir $temp_dir `
  "https://github.com/rust-marker/marker/releases/download/v$version/$files"

scripts/release/install.sh

@@ -14,6 +14,13 @@
 
 set -euo pipefail
 
+echo "Bash version: $BASH_VERSION" >&2
+
+# Configure the retries for downloading the release assets
+: "${MARKER_NET_RETRY_COUNT:=5}"
+: "${MARKER_NET_RETRY_MAX_DELAY:=60}"
+: "${RUSTUP_MAX_RETRIES:=$MARKER_NET_RETRY_COUNT}"
+
 # This script isn't meant to be run from `master`, but if it is, then
 # it will install the latest version be it a stable version or a pre-release.
 # region replace marker version unstable
@@ -31,12 +38,23 @@ function step {
  $cmd "$@"
 }
 
-echo "Bash version: $BASH_VERSION" >&2
+cat <<EOF
+Using config env vars (override these if needed):
+ MARKER_NET_RETRY_COUNT=$MARKER_NET_RETRY_COUNT
+ MARKER_NET_RETRY_MAX_DELAY=$MARKER_NET_RETRY_MAX_DELAY
+ RUSTUP_MAX_RETRIES=$RUSTUP_MAX_RETRIES
+EOF
 
 step curl --version
 step grep --version
 step tar --version
 
+curl_version=$(curl --version | grep -o 'curl [0-9]\+\.[0-9]\+\.[0-9]\+')
+curl_major=$(echo "$curl_version" | grep -o ' [0-9]\+\.' | grep -o '[0-9]\+')
+curl_minor=$(echo "$curl_version" | grep -o '\.[0-9]\+\.' | grep -o '[0-9]\+')
+
+echo "Parsed curl major.minor: $curl_major.$curl_minor"
+
 step rustup install --profile minimal --no-self-update $toolchain
 
 # MacOS uses some shitty BSD grep by default, and there isn't support for
@@ -61,6 +79,22 @@ function cleanup {
 
 files="{cargo-marker,marker_rustc_driver}-$host_triple.{tar.gz,sha256}"
 
+# We don't need such a condition in windows installation script because the
+# version of curl on windows 2019 is quite recent (8.2.1). However the GitHub
+# OS image for ubuntu-20.04 uses curl 7.68.0, and this is the environment where
+# this if condition is needed.
+if [[ "$curl_major" -lt 7 || $curl_major -eq 7 && "$curl_minor" -lt 71 ]]; then
+ echo -e "
+\033[33;1m[WARN] Installed curl version is $curl_major.$curl_minor, but \
+--retry-all-errors option is supported only since curl 7.71, so this option \
+will not be set. This means that if the download fails due to an error HTTP status \
+code, it won't be retried. The script will retry only 'connection refused' errors.\033[0m
+" >&2
+ retry_all_errors="--retry-connrefused"
+else
+ retry_all_errors="--retry-all-errors"
+fi
+
 # Download all files using a single TCP connection with HTTP2 multiplexing
 # Unfortunately, `curl 7.68.0` installed on Ubuntu 20.04 Github Actions runner
 # doesn't have the `--output-dir` option (it was added in `7.73.0`), so we have
@@ -71,8 +105,9 @@ step curl \
  --silent \
  --fail \
  --show-error \
- --retry 5 \
- --retry-connrefused \
+ --retry "$MARKER_NET_RETRY_COUNT" \
+ --retry-max-time "$MARKER_NET_RETRY_MAX_DELAY" \
+ $retry_all_errors \
  --remote-name \
  "https://github.com/rust-marker/marker/releases/download/v$version/$files"
 step popd

scripts/release/qemu-setup.sh

@@ -23,7 +23,7 @@ function curl_with_retry {
  --show-error \
  --fail \
  --retry 5 \
- --retry-connrefused \
+ --retry-all-errors \
  "$@"
 }
 

scripts/release/set-version.diff

@@ -131,11 +131,11 @@
 - - uses: rust-marker/marker@vX.Y
 + - uses: rust-marker/marker@v0.1
  with:
- --retry-connrefused \
+ --retry-all-errors \
 - https://raw.githubusercontent.com/rust-marker/marker/vX.Y/scripts/release/install.sh \
 + https://raw.githubusercontent.com/rust-marker/marker/v0.1/scripts/release/install.sh \
  | bash
- --retry-connrefused `
+ --retry-all-errors `
 - https://raw.githubusercontent.com/rust-marker/marker/vX.Y/scripts/release/install.ps1 `
 + https://raw.githubusercontent.com/rust-marker/marker/v0.1/scripts/release/install.ps1 `
  | powershell -command -