Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 1 addition & 6 deletions .evergreen/config.in.yml
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ functions:
MONGODB_URI="${MONGODB_URI}" \
AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \
SKIP_DEPS=${SKIP_DEPS|1} \
bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh

"run serverless tests":
Expand Down Expand Up @@ -291,7 +291,6 @@ functions:
AUTH=${AUTH} \
SSL=${SSL} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} \
SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
TOPOLOGY="${TOPOLOGY}" \
Expand All @@ -312,7 +311,6 @@ functions:
AUTH=${AUTH} \
SSL=${SSL} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} \
TOPOLOGY="${TOPOLOGY}" \
COMPRESSOR="${COMPRESSOR}" \
SKIP_DEPS=${SKIP_DEPS|1} \
Expand Down Expand Up @@ -457,9 +455,6 @@ functions:
${PREPARE_SHELL}
NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
- command: expansions.update
params:
file: src/deps-expansion.yml

"install aws-credential-providers":
- command: shell.exec
Expand Down
7 changes: 1 addition & 6 deletions .evergreen/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,7 @@ functions:
MONGODB_URI="${MONGODB_URI}" \
AUTH=${AUTH} SSL=${SSL} TEST_CSFLE=true \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} SKIP_DEPS=${SKIP_DEPS|1} \
SKIP_DEPS=${SKIP_DEPS|1} \
bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
run serverless tests:
- command: timeout.update
Expand Down Expand Up @@ -257,7 +257,6 @@ functions:
AUTH=${AUTH} \
SSL=${SSL} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} \
SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}" \
MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}" \
TOPOLOGY="${TOPOLOGY}" \
Expand All @@ -277,7 +276,6 @@ functions:
AUTH=${AUTH} \
SSL=${SSL} \
MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
NODE_VERSION=${NODE_VERSION} \
TOPOLOGY="${TOPOLOGY}" \
COMPRESSOR="${COMPRESSOR}" \
SKIP_DEPS=${SKIP_DEPS|1} \
Expand Down Expand Up @@ -407,9 +405,6 @@ functions:
${PREPARE_SHELL}
NODE_LTS_NAME=${NODE_LTS_NAME} NPM_OPTIONS=${NPM_OPTIONS}\
bash ${PROJECT_DIRECTORY}/.evergreen/install-dependencies.sh
- command: expansions.update
params:
file: src/deps-expansion.yml
install aws-credential-providers:
- command: shell.exec
type: setup
Expand Down
18 changes: 5 additions & 13 deletions .evergreen/init-nvm.sh
Original file line number Diff line number Diff line change
@@ -1,21 +1,13 @@
#! /usr/bin/env bash

export PATH="/opt/mongodbtoolchain/v2/bin:$PATH"
NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"

NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"
if [[ "$OS" == "Windows_NT" ]]; then
NVM_HOME=$(cygpath -w "$NVM_DIR")
export NVM_HOME
NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
export NVM_SYMLINK
NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
export NVM_ARTIFACTS_PATH
PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
export PATH
echo "updated path on windows PATH=$PATH"
else
[ -s "$NVM_DIR/nvm.sh" ] && source "$NVM_DIR/nvm.sh"
NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH")
fi

export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH"
hash -r

export NODE_OPTIONS="--trace-deprecation --trace-warnings"
174 changes: 82 additions & 92 deletions .evergreen/install-dependencies.sh
Original file line number Diff line number Diff line change
@@ -1,105 +1,95 @@
#!/bin/bash
#!/usr/bin/env bash
set -o errexit # Exit the script with error if any of the commands fail

NVM_WINDOWS_URL="https://github.com/coreybutler/nvm-windows/releases/download/1.1.9/nvm-noinstall.zip"
NVM_URL="https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh"

NODE_LTS_NAME=${NODE_LTS_NAME:-fermium}
NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY}/node-artifacts"

# this needs to be explicitly exported for the nvm install below
export NVM_DIR="${NODE_ARTIFACTS_PATH}/nvm"
export XDG_CONFIG_HOME=${NODE_ARTIFACTS_PATH}

# create node artifacts path if needed
mkdir -p "${NODE_ARTIFACTS_PATH}"

function node_lts_to_version() {
case $1 in
"fermium")
echo 14
;;
"gallium")
echo 16
;;
"hydrogen")
echo 18
;;
"iron")
echo 20
;;
"latest")
echo 'latest'
;;
*)
echo "Unsupported Node LTS version $1"
;;
esac
}

function latest_version_for_node_major() {
local __NODE_MAJOR_VERSION=$1
local NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest-v${__NODE_MAJOR_VERSION}.x/SHASUMS256.txt"

if [ $__NODE_MAJOR_VERSION == 'latest' ]
then
NODE_DOWNLOAD_URI="https://nodejs.org/download/release/latest/SHASUMS256.txt"
fi

# check that the requested version does exist
curl --silent --fail $NODE_DOWNLOAD_URI &> /dev/null

echo $(curl --retry 8 --retry-delay 5 --max-time 50 --silent -o- $NODE_DOWNLOAD_URI | head -n 1 | awk '{print $2};' | cut -d- -f2)
}

NODE_MAJOR_VERSION=$(node_lts_to_version $NODE_LTS_NAME)
NODE_VERSION=$(latest_version_for_node_major $NODE_MAJOR_VERSION)
NODE_VERSION=${NODE_VERSION:1} # :1 gets rid of the leading 'v'

echo "set version to $NODE_VERSION"

# output node version to expansions file for use in subsequent scripts
cat <<EOT > deps-expansion.yml
NODE_VERSION: "$NODE_VERSION"
EOT
NODE_ARTIFACTS_PATH="${PROJECT_DIRECTORY:-$(pwd)}/node-artifacts"
if [[ "$OS" = "Windows_NT" ]]; then NODE_ARTIFACTS_PATH=$(cygpath --unix "$NODE_ARTIFACTS_PATH"); fi

mkdir -p "$NODE_ARTIFACTS_PATH/npm_global"

# Comparisons are all case insensitive
shopt -s nocasematch

# index.tab is a sorted tab separated values file with the following headers
# 0 1 2 3 4 5 6 7 8 9 10
# version date files npm v8 uv zlib openssl modules lts security
curl --retry 8 -sS "https://nodejs.org/dist/index.tab" --max-time 300 --output node_index.tab
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a long script, could we consider breaking it into smaller functions? Or moving the logic that determines the download URL into a separate file? That might help with the readability and modularity of the script.

Copy link
Contributor Author

@nbbeeken nbbeeken Mar 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's shorter than the script that proceeded it, I'd like to keep it in one file to make the porting and future updating easier. Perhaps I could put functionality inside bash functions to organize it? but it will be longer as a result

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't really care about the length, if it's well organized. This file is just dense and the code is opaque. Some logical grouping might make it easier to work with in the future?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use bash functions, we can then source this locally and test each component

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is the green CI not testing enough? we can't proceed if any step in this script fails, which part is opaque, can I improve variable names?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of the logic related to finding the correct version of Node and downloading it is a little confusing, and since there's no clear boundary as to when it starts and stops, it's hard to tell when you're reading the script when that portion of code is finished. This is why we use functions, to logically group related functionality and to make understanding the higher-level purpose and logic of a script easier to GROK at a glance

From a CI maintenance standpoint, ideally we never touch this code again. But if we do have to, I'd rather that it was logically group so that it's easier to reason about at the high level without reading and thoroughly understand the entire script.

And yes, you're right, a green CI means the script is passing. But if we ever have to touch this again, it would be nice if we could source the file and test it in a piecemeal fashion, as opposed to testing the script in its entirety or commenting out large chunks of code to test parts of it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from slack, we're aligned on the shortness of the script and since we plan to copy it to many places we're agreed that its fine as is, we may in the future improve this if there's a significant change needed to support a platform we don't currently predict here


while IFS=$'\t' read -r -a row; do
node_index_version="${row[0]}"
node_index_date="${row[1]}"
node_index_lts="${row[9]}"
[[ "$node_index_version" = "version" ]] && continue # skip tsv header
[[ "$NODE_LTS_NAME" = "latest" ]] && break # first line is latest
[[ "$NODE_LTS_NAME" = "$node_index_lts" ]] && break # case insensitive compare
done < node_index.tab

if [[ "$OS" = "Windows_NT" ]]; then
operating_system="win"
elif [[ $(uname) = "darwin" ]]; then
operating_system="darwin"
elif [[ $(uname) = "linux" ]]; then
operating_system="linux"
else
echo "Unable to determine operating system: $operating_system"
exit 1
fi

# install Node.js on Windows
if [[ "$OS" == "Windows_NT" ]]; then
# Delete pre-existing node to avoid version conflicts
rm -rf "/cygdrive/c/Program Files/nodejs"
architecture=$(uname -m)
if [[ $architecture = "x86_64" ]]; then
architecture="x64"
elif [[ $architecture = "arm64" ]]; then
architecture="arm64"
elif [[ $architecture == s390* ]]; then
architecture="s390x"
elif [[ $architecture == ppc* ]]; then
architecture="ppc64le"
else
echo "Unable to determine operating system: $architecture"
exit 1
fi

file_extension="tar.gz"
if [[ "$OS" = "Windows_NT" ]]; then file_extension="zip"; fi

NVM_HOME=$(cygpath -w "$NVM_DIR")
export NVM_HOME
NVM_SYMLINK=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
export NVM_SYMLINK
NVM_ARTIFACTS_PATH=$(cygpath -w "$NODE_ARTIFACTS_PATH/bin")
export NVM_ARTIFACTS_PATH
PATH=$(cygpath $NVM_SYMLINK):$(cygpath $NVM_HOME):$PATH
export PATH
node_directory="node-${node_index_version}-${operating_system}-${architecture}"
node_archive="${node_directory}.${file_extension}"
node_archive_path="$NODE_ARTIFACTS_PATH/${node_archive}"
node_download_url="https://nodejs.org/dist/${node_index_version}/${node_archive}"

curl -L $NVM_WINDOWS_URL -o nvm.zip
unzip -d "$NVM_DIR" nvm.zip
rm nvm.zip
echo "Node.js ${node_index_version} for ${operating_system}-${architecture} released on ${node_index_date}"

chmod 777 "$NVM_DIR"
chmod -R a+rx "$NVM_DIR"
set -o xtrace

cat <<EOT > "$NVM_DIR/settings.txt"
root: $NVM_HOME
path: $NVM_SYMLINK
EOT
nvm install "$NODE_VERSION"
nvm use "$NODE_VERSION"
which node || echo "node not found, PATH=$PATH"
which npm || echo "npm not found, PATH=$PATH"
npm cache clear --force # Fixes: Cannot read properties of null (reading 'pickAlgorithm') error on windows
curl --fail --retry 8 -sS "${node_download_url}" --max-time 300 --output "$node_archive_path"

# install Node.js on Linux/MacOS
if [[ "$file_extension" = "zip" ]]; then
unzip -q "$node_archive_path" -d "${NODE_ARTIFACTS_PATH}"
mkdir -p "${NODE_ARTIFACTS_PATH}/nodejs"
# Windows "bins" are at the top level
mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs/bin"
# Need to add executable flag ourselves
chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/node.exe"
chmod +x "${NODE_ARTIFACTS_PATH}/nodejs/bin/npm"
else
curl -o- $NVM_URL | bash
[ -s "${NVM_DIR}/nvm.sh" ] && source "${NVM_DIR}/nvm.sh"
nvm install --no-progress "$NODE_VERSION"
tar -xf "$node_archive_path" -C "${NODE_ARTIFACTS_PATH}"
mv "${NODE_ARTIFACTS_PATH}/${node_directory}" "${NODE_ARTIFACTS_PATH}/nodejs"
fi

npm install ${NPM_OPTIONS}
export PATH="$NODE_ARTIFACTS_PATH/npm_global/bin:$NODE_ARTIFACTS_PATH/nodejs/bin:$PATH"
hash -r

# Set npm -g prefix to our local artifacts directory
cat <<EOT > .npmrc
prefix=$NODE_ARTIFACTS_PATH/npm_global
EOT

if [[ $operating_system != "win" ]]; then
# Update npm to latest when we can
npm install --global npm@latest
hash -r
fi

echo "npm version: $(npm -v)"

npm install "${NPM_OPTIONS}"
2 changes: 1 addition & 1 deletion .evergreen/run-benchmarks.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#! /bin/bash

[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"

export MONGODB_URI=$MONGODB_URI

Expand Down
2 changes: 1 addition & 1 deletion .evergreen/run-custom-csfle-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ export CSFLE_KMS_PROVIDERS=${CSFLE_KMS_PROVIDERS}
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
echo "csfle CRYPT_SHARED_LIB_PATH: $CRYPT_SHARED_LIB_PATH"

[ -s "$PROJECT_DIRECTORY/node-artifacts/nvm/nvm.sh" ] && source "$PROJECT_DIRECTORY"/node-artifacts/nvm/nvm.sh
source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"

set -o xtrace # Write all commands first to stderr
set -o errexit # Exit the script with error if any of the commands fail
Expand Down
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,7 @@ etc/docs/build
!docs/**/*.css
!docs/**/*.js
.nvmrc

node_index.tab
node-artifacts
.npmrc