From 16515a8d09a893a31378b23cb904e8dfb604fdca Mon Sep 17 00:00:00 2001 From: Eric Swanson <64809312+ericswanson-dfinity@users.noreply.github.com> Date: Fri, 2 Feb 2024 16:02:47 -0800 Subject: [PATCH] chore(dfxvm-installer): install script maintenance (#3566) * remove get_tag_from_manifest_json * remove unused function * copy check_help_for and downloader from rustup * rework tarball extraction to avoid rm -rf --- public/install-dfxvm.sh | 287 ++++++++++++++++++++++++++++++---------- 1 file changed, 216 insertions(+), 71 deletions(-) diff --git a/public/install-dfxvm.sh b/public/install-dfxvm.sh index fa33206168..54f85f4d61 100644 --- a/public/install-dfxvm.sh +++ b/public/install-dfxvm.sh @@ -11,24 +11,6 @@ ## set -u -# Functions useful for dealing with the manifest (which is JSON). - -# Get the version of a tag from the manifest JSON file. -# Arguments: -# $1 - The tag to get. -# STDIN - The manifest file. -# Returns: -# 0 if the tag was found, 1 if it wasn't. -# Prints out the version number. -get_tag_from_manifest_json() { - # Find the tag in the file. Then get the last digits. - # The first grep returns `"tag_name": "1.2.3` (without the last quote). - cat \ - | tr -d '\n' \ - | grep -o "\"$1\":[[:space:]]*\"[a-zA-Z0-9.]*" \ - | grep -o "[0-9.]*$" -} - # A newline separated list of boolean flags. See the read_flags function to see how it's parsed. DFX_BOOL_FLAGS="" @@ -111,7 +93,6 @@ err() { say "$1" >&2 exit 1 } -## 110_assert.sh need_cmd() { if ! check_cmd "$1"; then @@ -144,61 +125,81 @@ ignore() { define_flag_BOOL "insecure" "Allows downloading from insecure URLs, either using HTTP or TLS 1.2 or less." check_help_for() { + local _arch local _cmd local _arg - local _ok + _arch="$1" + shift _cmd="$1" - _ok="y" shift - # If we're running on OS-X, older than 10.13, then we always - # fail to find these options to force fallback - if check_cmd sw_vers; then - case "$(sw_vers -productVersion)" in - 10.15*) ;; # Catalina - 11.*) ;; # Big Sur - 12.*) ;; # Monterey - 13.*) ;; # Ventura - *) - warn "Detected OS X platform older than 10.15 (Catalina)" - _ok="n" - ;; - esac + local _category + if "$_cmd" --help | grep -q 'For all options use the manual or "--help all".'; then + _category="all" + else + _category="" fi + case "$_arch" in + + *darwin*) + if check_cmd sw_vers; then + case $(sw_vers -productVersion) in + 10.*) + # If we're running on macOS, older than 10.13, then we always + # fail to find these options to force fallback + if [ "$(sw_vers -productVersion | cut -d. -f2)" -lt 13 ]; then + # Older than 10.13 + echo "Warning: Detected macOS platform older than 10.13" + return 1 + fi + ;; + + # We assume these will be OK for now + 11.*) ;; # Big Sur + 12.*) ;; # Monterey + 13.*) ;; # Ventura + 14.*) ;; # Sonoma + + *) + # Unknown product version, warn and continue + echo "Warning: Detected unknown macOS major version: $(sw_vers -productVersion)" + echo "Warning TLS capabilities detection may fail" + ;; + esac + fi + ;; + + esac + for _arg in "$@"; do - if ! "$_cmd" --help all | grep -q -- "$_arg"; then - _ok="n" + if ! "$_cmd" --help "$_category" | grep -q -- "$_arg"; then + return 1 fi done - test "$_ok" = "y" + true # not strictly needed } -# Check for an error message in the output of a command. -# Arguments: -# $1 - The error message to look for. -# $2... - The command and arguments to run. -# Returns: -# Whether false if the error message was not found, or true if it wasn't (so the feature is -# supported. -# TODO: move this logic to execute once during install.sh run. -check_support_for() { - local err="$1" - shift - local cmd="$*" - - # Run the command, grep for the error message, if it is found returns false, if it - # is not found, returns true. - ! ($cmd 2>&1 | grep "$err" >/dev/null) +is_zsh() { + [ -n "${ZSH_VERSION-}" ] } -# This wraps curl or wget. Try curl first, if not installed, use wget instead. +# This wraps curl or wget. Try curl first, if not installed, +# use wget instead. # Arguments: # $1 - URL to download. # $2 - Path to output the download. Use - to output to stdout. +# $3 - The architecture, used to determine TLS capabilities. downloader() { + # zsh does not split words by default, Required for curl retry arguments below. + is_zsh && setopt local_options shwordsplit + local _dld + local _ciphersuites + local _err + local _status + local _retry if check_cmd curl; then _dld=curl elif check_cmd wget; then @@ -210,28 +211,171 @@ downloader() { if [ "$1" = --check ]; then need_cmd "$_dld" elif [ "$_dld" = curl ]; then - if check_help_for curl --proto --tlsv1.2; then - curl --proto '=https' --tlsv1.2 --show-error --fail --connect-timeout 10 --retry 5 --location "$1" --output "$2" --progress-bar - elif ! [ "$flag_INSECURE" ]; then - warn "Not forcing TLS v1.2, this is potentially less secure" - curl --show-error --fail --connect-timeout 10 --retry 5 --location "$1" --output "$2" --progress-bar + check_curl_for_retry_support + _retry="$RETVAL" + get_ciphersuites_for_curl + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + # shellcheck disable=SC2086 # $retry is intentionally split + _err=$(curl $_retry --proto '=https' --tlsv1.2 --ciphers "$_ciphersuites" --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? else - err "TLS 1.2 is not supported on this platform. To force using it, use the --insecure flag." + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" curl --proto --tlsv1.2; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + # shellcheck disable=SC2086 # $retry is intentionally split + _err=$(curl $_retry --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + else + # shellcheck disable=SC2086 # $retry is intentionally split + _err=$(curl $_retry --proto '=https' --tlsv1.2 --silent --show-error --fail --location "$1" --output "$2" 2>&1) + _status=$? + fi fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q 404$; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status elif [ "$_dld" = wget ]; then - if check_help_for wget --https-only --secure-protocol; then - wget --https-only --secure-protocol=TLSv1_2 --timeout 10 --tries 5 --waitretry 5 "$1" -O "$2" - elif ! [ "$flag_INSECURE" ]; then - warn "Not forcing TLS v1.2, this is potentially less secure" - wget --timeout 10 --tries 5 --waitretry 5 "$1" -O "$2" + if [ "$(wget -V 2>&1 | head -2 | tail -1 | cut -f1 -d" ")" = "BusyBox" ]; then + echo "Warning: using the BusyBox version of wget. Not enforcing strong cipher suites for TLS or TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? else - err "TLS 1.2 is not supported on this platform. To force using it, use the --insecure flag." + get_ciphersuites_for_wget + _ciphersuites="$RETVAL" + if [ -n "$_ciphersuites" ]; then + _err=$(wget --https-only --secure-protocol=TLSv1_2 --ciphers "$_ciphersuites" "$1" -O "$2" 2>&1) + _status=$? + else + echo "Warning: Not enforcing strong cipher suites for TLS, this is potentially less secure" + if ! check_help_for "$3" wget --https-only --secure-protocol; then + echo "Warning: Not enforcing TLS v1.2, this is potentially less secure" + _err=$(wget "$1" -O "$2" 2>&1) + _status=$? + else + _err=$(wget --https-only --secure-protocol=TLSv1_2 "$1" -O "$2" 2>&1) + _status=$? + fi + fi fi + if [ -n "$_err" ]; then + echo "$_err" >&2 + if echo "$_err" | grep -q ' 404 Not Found$'; then + err "installer for platform '$3' not found, this may be unsupported" + fi + fi + return $_status else err "Unknown downloader" # should not reach here fi } +# Check if curl supports the --retry flag, then pass it to the curl invocation. +check_curl_for_retry_support() { + local _retry_supported="" + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--retry"; then + _retry_supported="--retry 3" + if check_help_for "notspecified" "curl" "--continue-at"; then + # "-C -" tells curl to automatically find where to resume the download when retrying. + _retry_supported="--retry 3 -C -" + fi + fi + + RETVAL="$_retry_supported" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these curl backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_curl() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _openssl_syntax="no" + local _gnutls_syntax="no" + local _backend_supported="yes" + if curl -V | grep -q ' OpenSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' LibreSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' BoringSSL/'; then + _openssl_syntax="yes" + elif curl -V | grep -iq ' GnuTLS/'; then + _gnutls_syntax="yes" + else + _backend_supported="no" + fi + + local _args_supported="no" + if [ "$_backend_supported" = "yes" ]; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "curl" "--tlsv1.2" "--ciphers" "--proto"; then + _args_supported="yes" + fi + fi + + local _cs="" + if [ "$_args_supported" = "yes" ]; then + if [ "$_openssl_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "openssl") + elif [ "$_gnutls_syntax" = "yes" ]; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return cipher suite string specified by user, otherwise return strong TLS 1.2-1.3 cipher suites +# if support by local tools is detected. Detection currently supports these wget backends: +# GnuTLS and OpenSSL (possibly also LibreSSL and BoringSSL). Return value can be empty. +get_ciphersuites_for_wget() { + if [ -n "${RUSTUP_TLS_CIPHERSUITES-}" ]; then + # user specified custom cipher suites, assume they know what they're doing + RETVAL="$RUSTUP_TLS_CIPHERSUITES" + return + fi + + local _cs="" + if wget -V | grep -q '\-DHAVE_LIBSSL'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "openssl") + fi + elif wget -V | grep -q '\-DHAVE_LIBGNUTLS'; then + # "unspecified" is for arch, allows for possibility old OS using macports, homebrew, etc. + if check_help_for "notspecified" "wget" "TLSv1_2" "--ciphers" "--https-only" "--secure-protocol"; then + _cs=$(get_strong_ciphersuites_for "gnutls") + fi + fi + + RETVAL="$_cs" +} + +# Return strong TLS 1.2-1.3 cipher suites in OpenSSL or GnuTLS syntax. TLS 1.2 +# excludes non-ECDHE and non-AEAD cipher suites. DHE is excluded due to bad +# DH params often found on servers (see RFC 7919). Sequence matches or is +# similar to Firefox 68 ESR with weak cipher suites disabled via about:config. +# $1 must be openssl or gnutls. +get_strong_ciphersuites_for() { + if [ "$1" = "openssl" ]; then + # OpenSSL is forgiving of unknown values, no problems with TLS 1.3 values on versions that don't support it yet. + echo "TLS_AES_128_GCM_SHA256:TLS_CHACHA20_POLY1305_SHA256:TLS_AES_256_GCM_SHA384:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384" + elif [ "$1" = "gnutls" ]; then + # GnuTLS isn't forgiving of unknown values, so this may require a GnuTLS version that supports TLS 1.3 even if wget doesn't. + # Begin with SECURE128 (and higher) then remove/add to build cipher suites. Produces same 9 cipher suites as OpenSSL but in slightly different order. + echo "SECURE128:-VERS-SSL3.0:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-DTLS-ALL:-CIPHER-ALL:-MAC-ALL:-KX-ALL:+AEAD:+ECDHE-ECDSA:+ECDHE-RSA:+AES-128-GCM:+CHACHA20-POLY1305:+AES-256-GCM" + fi +} + DFXVM_GITHUB_LATEST_RELEASE_ROOT="${DFXVM_GITHUB_LATEST_RELEASE_ROOT:-https://github.com/dfinity/dfxvm/releases/latest/download}" DFX_VERSION="${DFX_VERSION-}" @@ -252,14 +396,13 @@ download_and_install() { local _sha256_url="${_tarball_url}.sha256" log "Downloading latest release..." - ensure downloader "$_tarball_url" "${_tarball_filename}" - ensure downloader "$_sha256_url" "${_sha256_filename}" + ensure downloader "$_tarball_url" "${_tarball_filename}" "$_arch" + ensure downloader "$_sha256_url" "${_sha256_filename}" "$_arch" log "Checking integrity of tarball..." ensure "$SHASUM" -c "${_sha256_filename}" - ensure tar -xzf "${_tarball_filename}" - ensure cd "${_archive}" >/dev/null + ensure tar -xzf "${_tarball_filename}" --strip-components=1 "${_archive}/dfxvm" ensure chmod u+x dfxvm ensure mv dfxvm dfxvm-init @@ -320,7 +463,9 @@ main() { ) local _subshell_exit_code=$? - ignore rm -rf "${_dir}" + ignore rm "${_dir}"/dfxvm* + ignore rmdir "${_dir}" + exit $_subshell_exit_code }