From 31fda6a602957d38ed1fed1e87b567ce6bd463a4 Mon Sep 17 00:00:00 2001 From: Andrea Fassina Date: Sat, 20 May 2023 17:27:39 +0200 Subject: [PATCH] tools: log and verify sha256sum --- tools/dep_updaters/update-ada.sh | 6 +++++- tools/dep_updaters/update-base64.sh | 10 +++++++++- tools/dep_updaters/update-brotli.sh | 8 ++++++-- tools/dep_updaters/update-c-ares.sh | 8 +++++++- tools/dep_updaters/update-libuv.sh | 10 +++++++++- tools/dep_updaters/update-llhttp.sh | 14 ++++++++----- tools/dep_updaters/update-nghttp2.sh | 8 ++++++++ tools/dep_updaters/update-nghttp3.sh | 4 ++++ tools/dep_updaters/update-ngtcp2.sh | 4 ++++ tools/dep_updaters/update-npm.sh | 7 ++++++- tools/dep_updaters/update-openssl.sh | 8 +++++++- tools/dep_updaters/update-simdutf.sh | 6 +++++- tools/dep_updaters/update-uvwasi.sh | 5 +++++ tools/dep_updaters/update-zlib.sh | 9 +++++++-- tools/dep_updaters/utils.sh | 30 ++++++++++++++++++++++++++++ 15 files changed, 121 insertions(+), 16 deletions(-) create mode 100644 tools/dep_updaters/utils.sh diff --git a/tools/dep_updaters/update-ada.sh b/tools/dep_updaters/update-ada.sh index a9aa64731dc344..a26f854a1b2c52 100755 --- a/tools/dep_updaters/update-ada.sh +++ b/tools/dep_updaters/update-ada.sh @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/ada-url/ada/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -37,13 +40,14 @@ cleanup () { trap cleanup INT TERM EXIT ADA_REF="v$NEW_VERSION" -ADA_ZIP="ada-$NEW_VERSION.zip" +ADA_ZIP="ada-$ADA_REF.zip" ADA_LICENSE="LICENSE-MIT" cd "$WORKSPACE" echo "Fetching ada source archive..." curl -sL -o "$ADA_ZIP" "https://github.com/ada-url/ada/releases/download/$ADA_REF/singleheader.zip" +log_and_verify_sha256sum "ada" "$ADA_ZIP" unzip "$ADA_ZIP" rm "$ADA_ZIP" diff --git a/tools/dep_updaters/update-base64.sh b/tools/dep_updaters/update-base64.sh index b0d8693f82f1b7..ff39a5f6e82010 100755 --- a/tools/dep_updaters/update-base64.sh +++ b/tools/dep_updaters/update-base64.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/aklomp/base64/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -39,8 +42,13 @@ trap cleanup INT TERM EXIT cd "$WORKSPACE" +BASE64_TARBALL="base64-v$NEW_VERSION.tar.gz" + echo "Fetching base64 source archive" -curl -sL "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION" | tar xzf - +curl -sL -o "$BASE64_TARBALL" "https://api.github.com/repos/aklomp/base64/tarball/v$NEW_VERSION" +log_and_verify_sha256sum "base64" "$BASE64_TARBALL" +gzip -dc "$BASE64_TARBALL" | tar xf - +rm "$BASE64_TARBALL" mv aklomp-base64-* base64 echo "Replacing existing base64" diff --git a/tools/dep_updaters/update-brotli.sh b/tools/dep_updaters/update-brotli.sh index 651ae57d3f2e62..3e9d6eddeaf665 100755 --- a/tools/dep_updaters/update-brotli.sh +++ b/tools/dep_updaters/update-brotli.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/google/brotli/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -44,10 +47,11 @@ trap cleanup INT TERM EXIT cd "$WORKSPACE" -BROTLI_TARBALL="v$NEW_VERSION.tar.gz" +BROTLI_TARBALL="brotli-v$NEW_VERSION.tar.gz" echo "Fetching brotli source archive" -curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/$BROTLI_TARBALL" +curl -sL -o "$BROTLI_TARBALL" "https://github.com/google/brotli/archive/v$NEW_VERSION.tar.gz" +log_and_verify_sha256sum "brotli" "$BROTLI_TARBALL" gzip -dc "$BROTLI_TARBALL" | tar xf - rm "$BROTLI_TARBALL" mv "brotli-$NEW_VERSION" "brotli" diff --git a/tools/dep_updaters/update-c-ares.sh b/tools/dep_updaters/update-c-ares.sh index 79d964e61f08a5..4bef7d20abca19 100755 --- a/tools/dep_updaters/update-c-ares.sh +++ b/tools/dep_updaters/update-c-ares.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/c-ares/c-ares/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -43,7 +46,10 @@ ARES_TARBALL="c-ares-$NEW_VERSION.tar.gz" cd "$WORKSPACE" echo "Fetching c-ares source archive" -curl -sL "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL" | tar xz +curl -sL -o "$ARES_TARBALL" "https://github.com/c-ares/c-ares/releases/download/$ARES_REF/$ARES_TARBALL" +log_and_verify_sha256sum "c-ares" "$ARES_TARBALL" +gzip -dc "$ARES_TARBALL" | tar xf - +rm "$ARES_TARBALL" mv "c-ares-$NEW_VERSION" cares echo "Removing tests" diff --git a/tools/dep_updaters/update-libuv.sh b/tools/dep_updaters/update-libuv.sh index 1a0b2c8efc7c89..ed4d36170451b8 100755 --- a/tools/dep_updaters/update-libuv.sh +++ b/tools/dep_updaters/update-libuv.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/libuv/libuv/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -46,8 +49,13 @@ trap cleanup INT TERM EXIT cd "$WORKSPACE" +LIBUV_TARBALL="libuv-v$NEW_VERSION.tar.gz" + echo "Fetching libuv source archive..." -curl -sL "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION" | tar xzf - +curl -sL -o "$LIBUV_TARBALL" "https://api.github.com/repos/libuv/libuv/tarball/v$NEW_VERSION" +log_and_verify_sha256sum "libuv" "$LIBUV_TARBALL" +gzip -dc "$LIBUV_TARBALL" | tar xf - +rm "$LIBUV_TARBALL" mv libuv-libuv-* uv echo "Replacing existing libuv (except GYP build files)" diff --git a/tools/dep_updaters/update-llhttp.sh b/tools/dep_updaters/update-llhttp.sh index 9c46536f205b40..30fb06667ece5b 100755 --- a/tools/dep_updaters/update-llhttp.sh +++ b/tools/dep_updaters/update-llhttp.sh @@ -9,6 +9,9 @@ DEPS_DIR="${BASE_DIR}/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/nodejs/llhttp/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -52,19 +55,20 @@ if echo "$NEW_VERSION" | grep -qs "/" ; then # Download a release echo "Checking out branch $BRANCH ..." git checkout "$BRANCH" - echo "Building llhtttp ..." + echo "Building llhttp ..." npm install make release - echo "Copying llhtttp release ..." + echo "Copying llhttp release ..." rm -rf "$DEPS_DIR/llhttp" cp -a release "$DEPS_DIR/llhttp" else echo "Download llhttp release $NEW_VERSION ..." - curl -sL -o llhttp.tar.gz "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz" - gzip -dc llhttp.tar.gz | tar xf - + LLHTTP_TARBALL="llhttp-v$NEW_VERSION.tar.gz" + curl -sL -o "$LLHTTP_TARBALL" "https://github.com/nodejs/llhttp/archive/refs/tags/release/v$NEW_VERSION.tar.gz" + gzip -dc "$LLHTTP_TARBALL" | tar xf - - echo "Copying llhtttp release ..." + echo "Copying llhttp release ..." rm -rf "$DEPS_DIR/llhttp" cp -a "llhttp-release-v$NEW_VERSION" "$DEPS_DIR/llhttp" fi diff --git a/tools/dep_updaters/update-nghttp2.sh b/tools/dep_updaters/update-nghttp2.sh index c53a620ba096ec..5ee7f1f08da0a2 100755 --- a/tools/dep_updaters/update-nghttp2.sh +++ b/tools/dep_updaters/update-nghttp2.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/nghttp2/nghttp2/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -44,6 +47,11 @@ cd "$WORKSPACE" echo "Fetching nghttp2 source archive" curl -sL -o "$NGHTTP2_TARBALL" "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/$NGHTTP2_TARBALL" + +DEPOSITED_CHECKSUM=$(curl -sL "https://github.com/nghttp2/nghttp2/releases/download/$NGHTTP2_REF/checksums.txt" | grep "$NGHTTP2_TARBALL") + +log_and_verify_sha256sum "nghttp2" "$NGHTTP2_TARBALL" "$DEPOSITED_CHECKSUM" + gzip -dc "$NGHTTP2_TARBALL" | tar xf - rm "$NGHTTP2_TARBALL" mv "nghttp2-$NEW_VERSION" nghttp2 diff --git a/tools/dep_updaters/update-nghttp3.sh b/tools/dep_updaters/update-nghttp3.sh index a3c035d871774b..f10165960dabae 100755 --- a/tools/dep_updaters/update-nghttp3.sh +++ b/tools/dep_updaters/update-nghttp3.sh @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/ngtcp2/nghttp3/releases'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -44,6 +47,7 @@ cd "$WORKSPACE" echo "Fetching nghttp3 source archive..." curl -sL -o "$NGHTTP3_ZIP.zip" "https://github.com/ngtcp2/nghttp3/archive/refs/tags/$NGHTTP3_REF.zip" +log_and_verify_sha256sum "nghttp3" "$NGHTTP3_ZIP.zip" unzip "$NGHTTP3_ZIP.zip" rm "$NGHTTP3_ZIP.zip" mv "$NGHTTP3_ZIP" nghttp3 diff --git a/tools/dep_updaters/update-ngtcp2.sh b/tools/dep_updaters/update-ngtcp2.sh index 0e7d43cb4ce0d7..9e9803ee6197e6 100755 --- a/tools/dep_updaters/update-ngtcp2.sh +++ b/tools/dep_updaters/update-ngtcp2.sh @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/ngtcp2/ngtcp2/releases'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -44,6 +47,7 @@ cd "$WORKSPACE" echo "Fetching ngtcp2 source archive..." curl -sL -o "$NGTCP2_ZIP.zip" "https://github.com/ngtcp2/ngtcp2/archive/refs/tags/$NGTCP2_REF.zip" +log_and_verify_sha256sum "ngtcp2" "$NGTCP2_ZIP.zip" unzip "$NGTCP2_ZIP.zip" rm "$NGTCP2_ZIP.zip" mv "$NGTCP2_ZIP" ngtcp2 diff --git a/tools/dep_updaters/update-npm.sh b/tools/dep_updaters/update-npm.sh index 9706bbfca85fe2..72aac6de1ce98f 100755 --- a/tools/dep_updaters/update-npm.sh +++ b/tools/dep_updaters/update-npm.sh @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NPM="$DEPS_DIR/npm/bin/npm-cli.js" NPM_VERSION=$1 @@ -30,12 +33,14 @@ trap cleanup INT TERM EXIT cd "$WORKSPACE" -NPM_TGZ=npm.tgz +NPM_TGZ="npm-v$NPM_VERSION.tar.gz" NPM_TARBALL="$($NODE "$NPM" view npm@"$NPM_VERSION" dist.tarball)" curl -s "$NPM_TARBALL" > "$NPM_TGZ" +log_and_verify_sha256sum "npm" "$NPM_TGZ" + rm -rf "$DEPS_DIR/npm" mkdir "$DEPS_DIR/npm" diff --git a/tools/dep_updaters/update-openssl.sh b/tools/dep_updaters/update-openssl.sh index 753120e54839bc..710bf3219aaf97 100755 --- a/tools/dep_updaters/update-openssl.sh +++ b/tools/dep_updaters/update-openssl.sh @@ -20,11 +20,17 @@ download() { echo "Making temporary workspace..." WORKSPACE=$(mktemp -d 2> /dev/null || mktemp -d -t 'tmp') + # shellcheck disable=SC1091 + . "$BASE_DIR/tools/dep_updaters/utils.sh" cd "$WORKSPACE" echo "Fetching OpenSSL source archive..." - curl -sL "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION" | tar xzf - + OPENSSL_TARBALL="openssl-v$OPENSSL_VERSION.tar.gz" + curl -sL -o "$OPENSSL_TARBALL" "https://api.github.com/repos/quictls/openssl/tarball/openssl-$OPENSSL_VERSION" + log_and_verify_sha256sum "openssl" "$OPENSSL_TARBALL" + gzip -dc "$OPENSSL_TARBALL" | tar xf - + rm "$OPENSSL_TARBALL" mv quictls-openssl-* openssl echo "Replacing existing OpenSSL..." diff --git a/tools/dep_updaters/update-simdutf.sh b/tools/dep_updaters/update-simdutf.sh index dba4ba49c62516..9eaa9f8149ef63 100755 --- a/tools/dep_updaters/update-simdutf.sh +++ b/tools/dep_updaters/update-simdutf.sh @@ -7,6 +7,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/simdutf/simdutf/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -36,13 +39,14 @@ cleanup () { trap cleanup INT TERM EXIT SIMDUTF_REF="v$NEW_VERSION" -SIMDUTF_ZIP="simdutf-$NEW_VERSION.zip" +SIMDUTF_ZIP="simdutf-$SIMDUTF_REF.zip" SIMDUTF_LICENSE="LICENSE-MIT" cd "$WORKSPACE" echo "Fetching simdutf source archive..." curl -sL -o "$SIMDUTF_ZIP" "https://github.com/simdutf/simdutf/releases/download/$SIMDUTF_REF/singleheader.zip" +log_and_verify_sha256sum "simdutf" "$SIMDUTF_ZIP" unzip "$SIMDUTF_ZIP" rm "$SIMDUTF_ZIP" rm ./*_demo.cpp diff --git a/tools/dep_updaters/update-uvwasi.sh b/tools/dep_updaters/update-uvwasi.sh index a6a66bf4e7672f..8ba9dbd9e1d150 100755 --- a/tools/dep_updaters/update-uvwasi.sh +++ b/tools/dep_updaters/update-uvwasi.sh @@ -8,6 +8,9 @@ DEPS_DIR="$BASE_DIR/deps" [ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" [ -x "$NODE" ] || NODE=$(command -v node) +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/nodejs/uvwasi/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); @@ -46,6 +49,8 @@ cd "$WORKSPACE" echo "Fetching UVWASI source archive..." curl -sL -o "$UVWASI_ZIP.zip" "https://github.com/nodejs/uvwasi/archive/refs/tags/v$NEW_VERSION.zip" +log_and_verify_sha256sum "uvwasi" "$UVWASI_ZIP.zip" + echo "Moving existing GYP build file" mv "$DEPS_DIR/uvwasi/"*.gyp "$WORKSPACE/" rm -rf "$DEPS_DIR/uvwasi/" diff --git a/tools/dep_updaters/update-zlib.sh b/tools/dep_updaters/update-zlib.sh index 3902e9221264b0..33e0a9b4552459 100755 --- a/tools/dep_updaters/update-zlib.sh +++ b/tools/dep_updaters/update-zlib.sh @@ -7,6 +7,9 @@ set -e BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd) DEPS_DIR="$BASE_DIR/deps" +# shellcheck disable=SC1091 +. "$BASE_DIR/tools/dep_updaters/utils.sh" + echo "Comparing latest upstream with current revision" git fetch https://chromium.googlesource.com/chromium/src/third_party/zlib.git HEAD @@ -49,10 +52,12 @@ cd "$WORKSPACE" mkdir zlib -ZLIB_TARBALL=zlib.tar.gz +ZLIB_TARBALL="zlib-v$NEW_VERSION.tar.gz" echo "Fetching zlib source archive" -curl -sL -o $ZLIB_TARBALL https://chromium.googlesource.com/chromium/src/+archive/refs/heads/main/third_party/$ZLIB_TARBALL +curl -sL -o "$ZLIB_TARBALL" https://chromium.googlesource.com/chromium/src/+archive/refs/heads/main/third_party/zlib.tar.gz + +log_and_verify_sha256sum "zlib" "$ZLIB_TARBALL" gzip -dc "$ZLIB_TARBALL" | tar xf - -C zlib/ diff --git a/tools/dep_updaters/utils.sh b/tools/dep_updaters/utils.sh new file mode 100644 index 00000000000000..21231e9410c6a8 --- /dev/null +++ b/tools/dep_updaters/utils.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +# This function logs the archive checksum and, if provided, compares it with +# the deposited checksum +# +# $1 is the package name e.g. 'acorn', 'ada', 'base64' etc. See that file +# for a complete list of package name +# $2 is the downloaded archive +# $3 (optional) is the deposited sha256 cheksum. When provided, it is checked +# against the checksum generated from the archive +log_and_verify_sha256sum() { + package_name="$1" + archive="$2" + checksum="$3" + bsd_formatted_checksum=$(sha256sum --tag "$archive") + if [ -z "$3" ]; then + echo "$bsd_formatted_checksum" + else + archive_checksum=$(sha256sum "$archive") + if [ "$checksum" = "$archive_checksum" ]; then + echo "Valid $package_name checksum" + echo "$bsd_formatted_checksum" + else + echo "ERROR - Invalid $package_name checksum:" + echo "deposited: $checksum" + echo "generated: $archive_checksum" + exit 1 + fi + fi +}