From 8c57c7cb3500c981224887634f78d903dfd188b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 20:05:42 +0100 Subject: [PATCH 01/16] Explicitly use Bash in scripts that actually use non-POSIX features or do not have a shebang at all --- .circleci/soltest.sh | 2 +- scripts/common_cmdline.sh | 1 + scripts/docker_deploy_manual.sh | 2 +- scripts/endToEndExtraction/create_traces.sh | 2 ++ scripts/install_obsolete_jsoncpp_1_7_4.sh | 2 +- scripts/uniqueErrors.sh | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.circleci/soltest.sh b/.circleci/soltest.sh index a0631fac2708..b6ac7313ec34 100755 --- a/.circleci/soltest.sh +++ b/.circleci/soltest.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#!/usr/bin/env bash #------------------------------------------------------------------------------ # Bash script to execute the Solidity tests by CircleCI. # diff --git a/scripts/common_cmdline.sh b/scripts/common_cmdline.sh index 5d579e81d3a1..5263b6121348 100644 --- a/scripts/common_cmdline.sh +++ b/scripts/common_cmdline.sh @@ -1,3 +1,4 @@ +#!/usr/bin/env bash # ------------------------------------------------------------------------------ # vim:ts=4:et # This file is part of solidity. diff --git a/scripts/docker_deploy_manual.sh b/scripts/docker_deploy_manual.sh index 4de739b31956..809f4caa55a9 100755 --- a/scripts/docker_deploy_manual.sh +++ b/scripts/docker_deploy_manual.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash set -e diff --git a/scripts/endToEndExtraction/create_traces.sh b/scripts/endToEndExtraction/create_traces.sh index 56ade255c835..8970c5cb30f4 100755 --- a/scripts/endToEndExtraction/create_traces.sh +++ b/scripts/endToEndExtraction/create_traces.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )" mkdir -p build diff --git a/scripts/install_obsolete_jsoncpp_1_7_4.sh b/scripts/install_obsolete_jsoncpp_1_7_4.sh index 1b7a5dbc4498..9dbbde49156c 100755 --- a/scripts/install_obsolete_jsoncpp_1_7_4.sh +++ b/scripts/install_obsolete_jsoncpp_1_7_4.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash set -e TEMPDIR=$(mktemp -d) diff --git a/scripts/uniqueErrors.sh b/scripts/uniqueErrors.sh index 07bb43c34da3..96083a1fe09d 100755 --- a/scripts/uniqueErrors.sh +++ b/scripts/uniqueErrors.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env bash REPO="$(dirname "$0")/.." From 4c186321c85164d00d55f3248e2b1190286f0017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 18:47:51 +0100 Subject: [PATCH 02/16] Replace some older shell syntax with newer and/or more reliable equivalents --- .circleci/soltest.sh | 2 +- scripts/ci/build_emscripten.sh | 2 +- scripts/create_source_tarball.sh | 4 ++-- scripts/docker_deploy_manual.sh | 2 +- scripts/tests.sh | 2 +- test/cmdlineTests.sh | 4 ++-- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/soltest.sh b/.circleci/soltest.sh index b6ac7313ec34..ed6708d16772 100755 --- a/.circleci/soltest.sh +++ b/.circleci/soltest.sh @@ -56,7 +56,7 @@ get_logfile_basename() { echo -ne "${filename}" } -BOOST_TEST_ARGS=("--color_output=no" "--show_progress=yes" "--logger=JUNIT,error,test_results/`get_logfile_basename`.xml" "${BOOST_TEST_ARGS[@]}") +BOOST_TEST_ARGS=("--color_output=no" "--show_progress=yes" "--logger=JUNIT,error,test_results/$(get_logfile_basename).xml" "${BOOST_TEST_ARGS[@]}") SOLTEST_ARGS=("--evm-version=$EVM" "${SOLTEST_FLAGS[@]}") test "${OPTIMIZE}" = "1" && SOLTEST_ARGS+=(--optimize) diff --git a/scripts/ci/build_emscripten.sh b/scripts/ci/build_emscripten.sh index 0e98487c6d9f..f4b8d734df80 100755 --- a/scripts/ci/build_emscripten.sh +++ b/scripts/ci/build_emscripten.sh @@ -78,6 +78,6 @@ mkdir -p upload cp "$BUILD_DIR/libsolc/soljson.js" upload/ cp "$BUILD_DIR/libsolc/soljson.js" ./ -OUTPUT_SIZE=`ls -la soljson.js` +OUTPUT_SIZE=$(ls -la soljson.js) echo "Emscripten output size: $OUTPUT_SIZE" diff --git a/scripts/create_source_tarball.sh b/scripts/create_source_tarball.sh index d75650a53e0d..df5c2967eb75 100755 --- a/scripts/create_source_tarball.sh +++ b/scripts/create_source_tarball.sh @@ -11,7 +11,7 @@ REPO_ROOT="$(dirname "$0")"/.. commitdate=$(git show --format=%ci HEAD | head -n 1 | cut - -b1-10 | sed -e 's/-0?/./' | sed -e 's/-0?/./') # file exists and has zero size -> not a prerelease - if [ -e prerelease.txt -a ! -s prerelease.txt ] + if [ -e prerelease.txt ] && [ ! -s prerelease.txt ] then versionstring="$version" else @@ -25,7 +25,7 @@ REPO_ROOT="$(dirname "$0")"/.. git checkout-index -a --prefix="$SOLDIR" # Store the commit hash echo "$commithash" > "$SOLDIR/commit_hash.txt" - if [ -e prerelease.txt -a ! -s prerelease.txt ] + if [ -e prerelease.txt ] && [ ! -s prerelease.txt ] then cp prerelease.txt "$SOLDIR/" fi diff --git a/scripts/docker_deploy_manual.sh b/scripts/docker_deploy_manual.sh index 809f4caa55a9..18d2228d0eeb 100755 --- a/scripts/docker_deploy_manual.sh +++ b/scripts/docker_deploy_manual.sh @@ -21,7 +21,7 @@ cd solidity commithash=$(git rev-parse --short=8 HEAD) echo -n "$commithash" > commit_hash.txt version=$("$(dirname "$0")/get_version.sh") -if [ "$branch" = "release" -o "$branch" = v"$version" ] +if [ "$branch" = "release" ] || [ "$branch" = v"$version" ] then echo -n > prerelease.txt else diff --git a/scripts/tests.sh b/scripts/tests.sh index 8014dcdae357..56979de652e8 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -34,7 +34,7 @@ IFS=" " read -r -a SMT_FLAGS <<< "$SMT_FLAGS" source "${REPO_ROOT}/scripts/common.sh" -WORKDIR=`mktemp -d` +WORKDIR=$(mktemp -d) CMDLINE_PID= cleanup() { diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 2aaeb058d8dc..b86561f71fc1 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -121,8 +121,8 @@ function test_solc_behaviour() local stderr_expected="${7}" local stdout_expectation_file="${8}" # the file to write to when user chooses to update stdout expectation local stderr_expectation_file="${9}" # the file to write to when user chooses to update stderr expectation - local stdout_path=`mktemp` - local stderr_path=`mktemp` + local stdout_path=$(mktemp) + local stderr_path=$(mktemp) trap "rm -f $stdout_path $stderr_path" EXIT From 1c7cc376d10e519abbf27058cdc2941df907e2c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 19:22:59 +0100 Subject: [PATCH 03/16] Use "$@" instead of $* --- scripts/uniqueErrors.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/uniqueErrors.sh b/scripts/uniqueErrors.sh index 96083a1fe09d..dd92fc28765f 100755 --- a/scripts/uniqueErrors.sh +++ b/scripts/uniqueErrors.sh @@ -4,7 +4,7 @@ REPO="$(dirname "$0")/.." echo "Finding unique failures..." ( -for x in $* +for x in "$@" do echo -n "$x" " # " # This subshell is a workaround to prevent the shell from printing From 75b87d159d5810aca77973203523bbac0ad8c923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 18:26:47 +0100 Subject: [PATCH 04/16] Declare variables separately when assigning the result of $() --- scripts/common_cmdline.sh | 4 ++-- scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh | 4 ++-- test/cmdlineTests.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/common_cmdline.sh b/scripts/common_cmdline.sh index 5263b6121348..2637fdedbb1a 100644 --- a/scripts/common_cmdline.sh +++ b/scripts/common_cmdline.sh @@ -51,12 +51,12 @@ function compileFull() local files=("$@") - local stderr_path=$(mktemp) + local stderr_path; stderr_path=$(mktemp) set +e "$SOLC" "${args[@]}" "${files[@]}" >/dev/null 2>"$stderr_path" local exit_code=$? - local errors=$(grep -v -E 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\|' < "$stderr_path") + local errors; errors=$(grep -v -E 'Warning: This is a pre-release compiler version|Warning: Experimental features are turned on|pragma experimental ABIEncoderV2|^ +--> |^ +\||^[0-9]+ +\|' < "$stderr_path") set -e rm "$stderr_path" diff --git a/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh b/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh index 3475057e4283..05ada07f3e9d 100755 --- a/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh +++ b/scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh @@ -86,7 +86,7 @@ function process_tag() { local TAG=$1 cd /src # Checkout the historic commit instead of the tag directly. - local HISTORIC_COMMIT_HASH="$(grep "${TAG}+" /tmp/release_commit_list.txt | cut -d '+' -f 2 | cut -d '.' -f 2)" + local HISTORIC_COMMIT_HASH; HISTORIC_COMMIT_HASH="$(grep "${TAG}+" /tmp/release_commit_list.txt | cut -d '+' -f 2 | cut -d '.' -f 2)" if [ "$(git cat-file -t "${HISTORIC_COMMIT_HASH}" 2>/dev/null)" == "commit" ]; then clean_git_checkout "$HISTORIC_COMMIT_HASH" else @@ -103,7 +103,7 @@ function process_tag() { VERSION=$(echo "$TAG" | cut -d v -f 2) fi - local COMMIT_HASH=$(git rev-parse --short=8 HEAD) + local COMMIT_HASH; COMMIT_HASH=$(git rev-parse --short=8 HEAD) local FULL_VERSION_SUFFIX="${TAG}+commit.${COMMIT_HASH}" local HISTORIC_VERSION_SUFFIX="${TAG}+commit.${HISTORIC_COMMIT_HASH}" diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index b86561f71fc1..16b5b5df69f4 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -121,8 +121,8 @@ function test_solc_behaviour() local stderr_expected="${7}" local stdout_expectation_file="${8}" # the file to write to when user chooses to update stdout expectation local stderr_expectation_file="${9}" # the file to write to when user chooses to update stderr expectation - local stdout_path=$(mktemp) - local stderr_path=$(mktemp) + local stdout_path; stdout_path=$(mktemp) + local stderr_path; stderr_path=$(mktemp) trap "rm -f $stdout_path $stderr_path" EXIT From d14c4288ec8c944953bed2590c3c3244dafc3b95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 20:08:45 +0100 Subject: [PATCH 05/16] Make scripts fail on non-zero return codes and undefined variables where possible --- scripts/check_style.sh | 6 ++++-- scripts/chk_shellscripts/chk_shellscripts.sh | 2 +- scripts/endToEndExtraction/create_traces.sh | 15 ++++++++------- scripts/install_cmake.sh | 2 +- scripts/install_obsolete_jsoncpp_1_7_4.sh | 2 +- scripts/install_static_z3.sh | 2 ++ scripts/soltest.sh | 2 +- 7 files changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/check_style.sh b/scripts/check_style.sh index b8033ea288c2..5c423230ce85 100755 --- a/scripts/check_style.sh +++ b/scripts/check_style.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -eu + ERROR_LOG="$(mktemp -t check_style_XXXXXX.log)" EXCLUDE_FILES=( @@ -20,7 +22,7 @@ REPO_ROOT="$(dirname "$0")"/.. cd "$REPO_ROOT" || exit 1 WHITESPACE=$(git grep -n -I -E "^.*[[:space:]]+$" | - grep -v "test/libsolidity/ASTJSON\|test/libsolidity/ASTRecoveryTests\|test/compilationTests/zeppelin/LICENSE\|${EXCLUDE_FILES_JOINED}" + grep -v "test/libsolidity/ASTJSON\|test/libsolidity/ASTRecoveryTests\|test/compilationTests/zeppelin/LICENSE\|${EXCLUDE_FILES_JOINED}" || true ) if [[ "$WHITESPACE" != "" ]] @@ -49,7 +51,7 @@ FORMATERROR=$( preparedGrep "[a-zA-Z0-9_]\s*[&][a-zA-Z_]" | grep -E -v "return [&]" # right-aligned reference ampersand (needs to exclude return) # right-aligned reference pointer star (needs to exclude return and comments) preparedGrep "[a-zA-Z0-9_]\s*[*][a-zA-Z_]" | grep -E -v -e "return [*]" -e "^* [*]" -e "^*//.*" -) | grep -E -v -e "^[a-zA-Z\./]*:[0-9]*:\s*\/(\/|\*)" -e "^test/" +) | grep -E -v -e "^[a-zA-Z\./]*:[0-9]*:\s*\/(\/|\*)" -e "^test/" || true ) if [[ "$FORMATERROR" != "" ]] diff --git a/scripts/chk_shellscripts/chk_shellscripts.sh b/scripts/chk_shellscripts/chk_shellscripts.sh index 12ece3f4ec7d..4f5c23743a4b 100755 --- a/scripts/chk_shellscripts/chk_shellscripts.sh +++ b/scripts/chk_shellscripts/chk_shellscripts.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -e +set -eu REPO_ROOT="$(dirname "$0")"/../.. REPO_ROOT=$(realpath "${REPO_ROOT}") diff --git a/scripts/endToEndExtraction/create_traces.sh b/scripts/endToEndExtraction/create_traces.sh index 8970c5cb30f4..3e1de4677c5d 100755 --- a/scripts/endToEndExtraction/create_traces.sh +++ b/scripts/endToEndExtraction/create_traces.sh @@ -1,24 +1,25 @@ #!/usr/bin/env bash +set -eu -BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P )" +BASE_PATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 && pwd -P )" mkdir -p build -cd build || exit +cd build cmake ../../../ make soltest -cd test/ || exit +cd test/ echo "running soltest on 'semanticTests/extracted'..." ./soltest --color_output=false --log_level=test_suite -t semanticTests/extracted/ -- --testpath "${BASE_PATH}/../../test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/extracted-tests.trace" echo "running soltest on 'semanticTests/extracted'... done" -cd "$BASE_PATH" || exit +cd "$BASE_PATH" git clone git@github.com:ethereum/solidity.git solidity-develop -cd solidity-develop || exit +cd solidity-develop mkdir -p build -cd build || exit +cd build cmake .. make soltest -cd test/ || exit +cd test/ echo "running soltest on 'SolidityEndToEndTest'..." ./soltest --color_output=false --log_level=test_suite -t SolidityEndToEndTest/ -- --testpath "${BASE_PATH}/solidity-develop/test" --no-smt --evmonepath /Users/alex/evmone/lib/libevmone.dylib --show-messages --show-metadata > "${BASE_PATH}/endToEndExtraction-tests.trace" echo "running soltest on 'SolidityEndToEndTest'... done" diff --git a/scripts/install_cmake.sh b/scripts/install_cmake.sh index d7317d1e557e..0533de7234e4 100755 --- a/scripts/install_cmake.sh +++ b/scripts/install_cmake.sh @@ -4,7 +4,7 @@ # (the cmake executable will be in ~/.local/bin). # This is mostly suitable for CIs, not end users. -set -e +set -eu VERSION_MAJOR=3 VERSION_MINOR=15 diff --git a/scripts/install_obsolete_jsoncpp_1_7_4.sh b/scripts/install_obsolete_jsoncpp_1_7_4.sh index 9dbbde49156c..eb617e651dfe 100755 --- a/scripts/install_obsolete_jsoncpp_1_7_4.sh +++ b/scripts/install_obsolete_jsoncpp_1_7_4.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -eu TEMPDIR=$(mktemp -d) ( diff --git a/scripts/install_static_z3.sh b/scripts/install_static_z3.sh index 6fba203d87a5..304ef410cfdf 100644 --- a/scripts/install_static_z3.sh +++ b/scripts/install_static_z3.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +set -eu + git clone --depth 1 --branch z3-4.8.7 https://github.com/Z3Prover/z3.git cd z3 mkdir build diff --git a/scripts/soltest.sh b/scripts/soltest.sh index db802aa4790f..e9b6de979f4f 100755 --- a/scripts/soltest.sh +++ b/scripts/soltest.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -eu REPO_ROOT="$(dirname "$0")"/.. USE_DEBUGGER=0 From 2784135c3767f631dfec698af45422bf9821683c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 00:18:03 +0100 Subject: [PATCH 06/16] Use read with the -r option --- test/cmdlineTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 16b5b5df69f4..db1a4f010a1e 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -91,7 +91,7 @@ function ask_expectation_update while true do - read -N 1 -p "(e)dit/(u)pdate expectations/(s)kip/(q)uit? " + read -r -N 1 -p "(e)dit/(u)pdate expectations/(s)kip/(q)uit? " echo case $REPLY in e*) "$editor" "$expectationFile"; break;; From a05f3a608dfb86646c30c88004b29fca5ba656da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Fri, 11 Dec 2020 20:01:09 +0100 Subject: [PATCH 07/16] Don't use $ on variables inside (()) --- .circleci/soltest_all.sh | 16 ++++++++-------- test/cmdlineTests.sh | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.circleci/soltest_all.sh b/.circleci/soltest_all.sh index 3aab9f19087b..fb738813bdfd 100755 --- a/.circleci/soltest_all.sh +++ b/.circleci/soltest_all.sh @@ -32,18 +32,18 @@ EVM_VALUES=(homestead byzantium constantinople petersburg istanbul) OPTIMIZE_VALUES=(0 1) STEPS=$(( 2 + ${#EVM_VALUES[@]} * ${#OPTIMIZE_VALUES[@]} )) -if (( $CIRCLE_NODE_TOTAL )) && (( $CIRCLE_NODE_TOTAL > 1 )) +if (( CIRCLE_NODE_TOTAL )) && (( CIRCLE_NODE_TOTAL > 1 )) then # Run step 1 as the only step on the first executor # and evenly distribute the other steps among # the other executors. # The first step takes much longer than the other steps. - if (( $CIRCLE_NODE_INDEX == 0 )) + if (( CIRCLE_NODE_INDEX == 0 )) then RUN_STEPS="1" else - export CIRCLE_NODE_INDEX=$(($CIRCLE_NODE_INDEX - 1)) - export CIRCLE_NODE_TOTAL=$(($CIRCLE_NODE_TOTAL - 1)) + export CIRCLE_NODE_INDEX=$((CIRCLE_NODE_INDEX - 1)) + export CIRCLE_NODE_TOTAL=$((CIRCLE_NODE_TOTAL - 1)) RUN_STEPS=$(seq 2 "$STEPS" | circleci tests split) fi else @@ -59,11 +59,11 @@ STEP=1 # Run SMTChecker tests separately, as the heaviest expected run. [[ " $RUN_STEPS " =~ " $STEP " ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t smtCheckerTests/*" "${REPODIR}/.circleci/soltest.sh" -STEP=$(($STEP + 1)) +STEP=$((STEP + 1)) # Run without SMTChecker tests. [[ " $RUN_STEPS " =~ " $STEP " ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t !smtCheckerTests" "${REPODIR}/.circleci/soltest.sh" -STEP=$(($STEP + 1)) +STEP=$((STEP + 1)) for OPTIMIZE in "${OPTIMIZE_VALUES[@]}" do @@ -74,11 +74,11 @@ do [ "${EVM}" = "byzantium" ] && [ "${OPTIMIZE}" = "0" ] && EWASM_ARGS="--ewasm" [[ " $RUN_STEPS " =~ " $STEP " ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" SOLTEST_FLAGS="$SOLTEST_FLAGS $EWASM_ARGS" BOOST_TEST_ARGS="-t !@nooptions" "${REPODIR}/.circleci/soltest.sh" - STEP=$(($STEP + 1)) + STEP=$((STEP + 1)) done done -if (($STEP != $STEPS + 1)) +if ((STEP != STEPS + 1)) then echo "Step counter not properly adjusted!" >&2 exit 1 diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index db1a4f010a1e..4c3d5dab96af 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -282,7 +282,7 @@ printTask "Running general commandline tests..." inputFiles="$(ls -1 "${tdir}/input."* 2> /dev/null || true)" inputCount="$(echo "${inputFiles}" | wc -w)" - if (( ${inputCount} > 1 )) + if (( inputCount > 1 )) then printError "Ambiguous input. Found input files in multiple formats:" echo -e "${inputFiles}" From dbfd3db03840fbdd843a87ca0e086bb6dc8e7480 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 03:05:06 +0100 Subject: [PATCH 08/16] Use mapfile instead of () for reading multiple lines as separate array items --- scripts/docs_version_pragma_check.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/docs_version_pragma_check.sh b/scripts/docs_version_pragma_check.sh index ca3c746f2a60..386f5e20d53e 100755 --- a/scripts/docs_version_pragma_check.sh +++ b/scripts/docs_version_pragma_check.sh @@ -66,11 +66,12 @@ function versionEqual() function getAllAvailableVersions() { allVersions=() - local allListedVersions=( $( + local allListedVersions + mapfile -t allListedVersions <<< "$( wget -q -O- https://binaries.soliditylang.org/bin/list.txt | grep -Po '(?<=soljson-v)\d+.\d+.\d+(?=\+commit)' | sort -V - ) ) + )" for listed in "${allListedVersions[@]}" do if versionGreater "$listed" "0.4.10" From 4e3ebda0f429328f32dba228c14bdebe587f7bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 01:26:11 +0100 Subject: [PATCH 09/16] Don't use negative exit codes --- scripts/ASTImportTest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index e69aa65b3d75..b03e11286db2 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -27,7 +27,7 @@ TESTED=0 if [ "$(ls | wc -l)" -ne 0 ]; then echo "Test directory not empty. Skipping!" - exit -1 + exit 1 fi # function tests whether exporting and importing again leaves the JSON ast unchanged From 62bf467fc2db4cab74fd1bb08950a2ef9b750fde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 01:28:00 +0100 Subject: [PATCH 10/16] Fix warnings about constructions that have no effect --- .circleci/soltest_all.sh | 7 ++----- scripts/ASTImportTest.sh | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.circleci/soltest_all.sh b/.circleci/soltest_all.sh index fb738813bdfd..ba6422692676 100755 --- a/.circleci/soltest_all.sh +++ b/.circleci/soltest_all.sh @@ -44,15 +44,12 @@ then else export CIRCLE_NODE_INDEX=$((CIRCLE_NODE_INDEX - 1)) export CIRCLE_NODE_TOTAL=$((CIRCLE_NODE_TOTAL - 1)) - RUN_STEPS=$(seq 2 "$STEPS" | circleci tests split) + RUN_STEPS=$(seq 2 "$STEPS" | circleci tests split | xargs) fi else - RUN_STEPS=$(seq "$STEPS") + RUN_STEPS=$(seq "$STEPS" | xargs) fi -# turn newlines into spaces -RUN_STEPS=$(echo $RUN_STEPS) - echo "Running steps $RUN_STEPS..." STEP=1 diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index b03e11286db2..d11b63c409e2 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -61,9 +61,9 @@ function testImportExportEquivalence { then echo -e "ERROR: JSONS differ for $1: \n $DIFF \n" echo "Expected:" - echo "$(cat ./expected.json)" + cat ./expected.json echo "Obtained:" - echo "$(cat ./obtained.json)" + cat ./obtained.json else # Use user supplied diff view binary $DIFFVIEW expected.json obtained.json From 7d845c02897cc63f9de018dfa5afceeb5122494c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 00:00:04 +0100 Subject: [PATCH 11/16] Don't use globs in command names --- scripts/wasm-rebuild/docker-scripts/rebuild_current.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh b/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh index 1160079b9ec4..41e554139e64 100755 --- a/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh +++ b/scripts/wasm-rebuild/docker-scripts/rebuild_current.sh @@ -41,7 +41,14 @@ if [ -d jsoncpp ]; then fi set +e -scripts/*/build_emscripten.sh + +if [ -e scripts/ci/build_emscripten.sh ]; then + scripts/ci/build_emscripten.sh +else + # The script used to be in scripts/ci/ in earlier versions. + scripts/travis-emscripten/build_emscripten.sh +fi + set -e mkdir -p upload From 04302d482828efb345e185db806a351abbf15bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 02:58:01 +0100 Subject: [PATCH 12/16] Use glob match instead of regex match when not actually using a regex --- .circleci/soltest_all.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/soltest_all.sh b/.circleci/soltest_all.sh index ba6422692676..143af71052ea 100755 --- a/.circleci/soltest_all.sh +++ b/.circleci/soltest_all.sh @@ -55,11 +55,11 @@ echo "Running steps $RUN_STEPS..." STEP=1 # Run SMTChecker tests separately, as the heaviest expected run. -[[ " $RUN_STEPS " =~ " $STEP " ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t smtCheckerTests/*" "${REPODIR}/.circleci/soltest.sh" +[[ " $RUN_STEPS " == *" $STEP "* ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t smtCheckerTests/*" "${REPODIR}/.circleci/soltest.sh" STEP=$((STEP + 1)) # Run without SMTChecker tests. -[[ " $RUN_STEPS " =~ " $STEP " ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t !smtCheckerTests" "${REPODIR}/.circleci/soltest.sh" +[[ " $RUN_STEPS " == *" $STEP "* ]] && EVM=istanbul OPTIMIZE=1 ABI_ENCODER_V1=1 BOOST_TEST_ARGS="-t !smtCheckerTests" "${REPODIR}/.circleci/soltest.sh" STEP=$((STEP + 1)) for OPTIMIZE in "${OPTIMIZE_VALUES[@]}" @@ -70,7 +70,7 @@ do EWASM_ARGS="" [ "${EVM}" = "byzantium" ] && [ "${OPTIMIZE}" = "0" ] && EWASM_ARGS="--ewasm" - [[ " $RUN_STEPS " =~ " $STEP " ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" SOLTEST_FLAGS="$SOLTEST_FLAGS $EWASM_ARGS" BOOST_TEST_ARGS="-t !@nooptions" "${REPODIR}/.circleci/soltest.sh" + [[ " $RUN_STEPS " == *" $STEP "* ]] && EVM="$EVM" OPTIMIZE="$OPTIMIZE" SOLTEST_FLAGS="$SOLTEST_FLAGS $EWASM_ARGS" BOOST_TEST_ARGS="-t !@nooptions" "${REPODIR}/.circleci/soltest.sh" STEP=$((STEP + 1)) done done From e8a70e8e4bc1ea5aa9585d8632ef801aad35f61f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 02:59:25 +0100 Subject: [PATCH 13/16] Prefix globs with ./ or use -- to make commands safe with files named like command-line options --- scripts/deps-ppa/static_z3.sh | 6 +++--- scripts/release_ppa.sh | 6 +++--- scripts/wasm-rebuild/docker-scripts/genbytecode.sh | 2 +- test/cmdlineTests.sh | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/deps-ppa/static_z3.sh b/scripts/deps-ppa/static_z3.sh index db335c0cda06..2d43913cc4b0 100755 --- a/scripts/deps-ppa/static_z3.sh +++ b/scripts/deps-ppa/static_z3.sh @@ -214,12 +214,12 @@ if wget --quiet -O $orig-tmp "$ppafilesurl/$orig" then echo "[WARN] Original tarball found in Ubuntu archive, using it instead" mv $orig-tmp $orig - new_size=$(ls -l *.orig.tar.gz | cut -d ' ' -f 5) + new_size=$(ls -l ./*.orig.tar.gz | cut -d ' ' -f 5) new_sha1=$(sha1sum $orig | cut -d ' ' -f 1) new_sha256=$(sha256sum $orig | cut -d ' ' -f 1) new_md5=$(md5sum $orig | cut -d ' ' -f 1) - sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" *.dsc - sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" *.changes + sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" ./*.dsc + sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" ./*.changes fi ) diff --git a/scripts/release_ppa.sh b/scripts/release_ppa.sh index da4187f42b65..72b34f66b3d0 100755 --- a/scripts/release_ppa.sh +++ b/scripts/release_ppa.sh @@ -274,12 +274,12 @@ if wget --quiet -O "$orig-tmp" "$ppafilesurl/$orig" then echo "[WARN] Original tarball found in Ubuntu archive, using it instead" mv "$orig-tmp" "$orig" - new_size=$(ls -l *.orig.tar.gz | cut -d ' ' -f 5) + new_size=$(ls -l ./*.orig.tar.gz | cut -d ' ' -f 5) new_sha1=$(sha1sum "$orig" | cut -d ' ' -f 1) new_sha256=$(sha256sum "$orig" | cut -d ' ' -f 1) new_md5=$(md5sum "$orig" | cut -d ' ' -f 1) - sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" *.dsc - sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" *.changes + sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" ./*.dsc + sed -i -e "s,$orig_sha1,$new_sha1,g" -e "s,$orig_sha256,$new_sha256,g" -e "s,$orig_size,$new_size,g" -e "s,$orig_md5,$new_md5,g" ./*.changes fi ) diff --git a/scripts/wasm-rebuild/docker-scripts/genbytecode.sh b/scripts/wasm-rebuild/docker-scripts/genbytecode.sh index f803f07831f8..d568c54526f6 100755 --- a/scripts/wasm-rebuild/docker-scripts/genbytecode.sh +++ b/scripts/wasm-rebuild/docker-scripts/genbytecode.sh @@ -93,6 +93,6 @@ for (var optimize of [false, true]) } EOF chmod +x solc - ./solc *.sol > /tmp/report.txt + ./solc -- *.sol > /tmp/report.txt ) rm -rf "$TMPDIR" diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index 4c3d5dab96af..d05768ebedbd 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -333,7 +333,7 @@ printTask "Compiling various other contracts and libraries..." do echo " - $dir" cd "$dir" - compileFull -w *.sol */*.sol + compileFull -w ./*.sol ./*/*.sol cd .. done ) @@ -499,8 +499,8 @@ SOLTMPDIR=$(mktemp -d) "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/test/ "$REPO_ROOT"/scripts/isolate_tests.py "$REPO_ROOT"/docs/ docs - echo *.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --quiet --input-files - echo *.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --without-optimizer --quiet --input-files + echo ./*.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --quiet --input-files + echo ./*.sol | xargs -P 4 -n 50 "${SOLIDITY_BUILD_DIR}/test/tools/solfuzzer" --without-optimizer --quiet --input-files ) rm -rf "$SOLTMPDIR" From 87f9eba1cfe59049fa5b8d758f141e6ff676fcfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 03:06:27 +0100 Subject: [PATCH 14/16] docs_version_pragma_check.sh: Set SOLC only for the duration of compileFull - This is a global variable and would be better to just make it a parameter. At least this makes shellcheck shut up --- scripts/docs_version_pragma_check.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/docs_version_pragma_check.sh b/scripts/docs_version_pragma_check.sh index 386f5e20d53e..22b3ddb87e27 100755 --- a/scripts/docs_version_pragma_check.sh +++ b/scripts/docs_version_pragma_check.sh @@ -188,8 +188,7 @@ SOLTMPDIR=$(mktemp -d) ln -sf "$solc_bin" "solc" chmod a+x solc - SOLC="$SOLTMPDIR/solc" - compileFull "${opts[@]}" "$SOLTMPDIR/$f" + SOLC="$SOLTMPDIR/solc" compileFull "${opts[@]}" "$SOLTMPDIR/$f" done ) rm -rf "$SOLTMPDIR" From c8fc241be5d8a245c05c7d76d3df733f64ed9ee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Sat, 12 Dec 2020 03:01:48 +0100 Subject: [PATCH 15/16] Avoid using $? in conditions and do && exit 1 instead if ! --- scripts/ASTImportTest.sh | 3 +-- test/cmdlineTests.sh | 24 +++++++----------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/scripts/ASTImportTest.sh b/scripts/ASTImportTest.sh index d11b63c409e2..28dbc5d588c6 100755 --- a/scripts/ASTImportTest.sh +++ b/scripts/ASTImportTest.sh @@ -45,8 +45,7 @@ function testImportExportEquivalence { # save exported json as expected result (silently) $SOLC --combined-json ast,compact-format --pretty-json "$nth_input_file" "${all_input_files[@]}" > expected.json 2> /dev/null # import it, and export it again as obtained result (silently) - $SOLC --import-ast --combined-json ast,compact-format --pretty-json expected.json > obtained.json 2> /dev/null - if [ $? -ne 0 ] + if ! $SOLC --import-ast --combined-json ast,compact-format --pretty-json expected.json > obtained.json 2> /dev/null then # For investigating, use exit 1 here so the script stops at the # first failing test diff --git a/test/cmdlineTests.sh b/test/cmdlineTests.sh index d05768ebedbd..56c453200498 100755 --- a/test/cmdlineTests.sh +++ b/test/cmdlineTests.sh @@ -378,7 +378,7 @@ echo "Done." printTask "Testing library checksum..." echo '' | "$SOLC" - --link --libraries a=0x90f20564390eAe531E810af625A22f51385Cd222 >/dev/null -! echo '' | "$SOLC" - --link --libraries a=0x80f20564390eAe531E810af625A22f51385Cd222 &>/dev/null +echo '' | "$SOLC" - --link --libraries a=0x80f20564390eAe531E810af625A22f51385Cd222 &>/dev/null && exit 1 printTask "Testing long library names..." echo '' | "$SOLC" - --link --libraries aveeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeerylonglibraryname=0x90f20564390eAe531E810af625A22f51385Cd222 >/dev/null @@ -408,7 +408,7 @@ SOLTMPDIR=$(mktemp -d) # First time it works echo 'contract C {} ' | "$SOLC" - --bin -o "$SOLTMPDIR/non-existing-stuff-to-create" 2>/dev/null # Second time it fails - ! echo 'contract C {} ' | "$SOLC" - --bin -o "$SOLTMPDIR/non-existing-stuff-to-create" 2>/dev/null + echo 'contract C {} ' | "$SOLC" - --bin -o "$SOLTMPDIR/non-existing-stuff-to-create" 2>/dev/null && exit 1 # Unless we force echo 'contract C {} ' | "$SOLC" - --overwrite --bin -o "$SOLTMPDIR/non-existing-stuff-to-create" 2>/dev/null ) @@ -422,8 +422,8 @@ printTask "Testing assemble, yul, strict-assembly and optimize..." # Test options above in conjunction with --optimize. # Using both, --assemble and --optimize should fail. - ! echo '{}' | "$SOLC" - --assemble --optimize &>/dev/null - ! echo '{}' | "$SOLC" - --yul --optimize &>/dev/null + echo '{}' | "$SOLC" - --assemble --optimize &>/dev/null && exit 1 + echo '{}' | "$SOLC" - --yul --optimize &>/dev/null && exit 1 # Test yul and strict assembly output # Non-empty code results in non-empty binary representation with optimizations turned off, @@ -451,24 +451,15 @@ SOLTMPDIR=$(mktemp -d) exit 1 fi - set +e - output=$(echo 'contract C {} ' | "$SOLC" - --bin 2>/dev/null | grep -q ":C") - result=$? - set -e - # The contract should be compiled - if [[ "$result" != 0 ]] + if ! output=$(echo 'contract C {} ' | "$SOLC" - --bin 2>/dev/null | grep -q ":C") then printError "Failed to compile a simple contract from standard input" exit 1 fi # This should not fail - set +e - output=$(echo '' | "$SOLC" --ast-compact-json - 2>/dev/null) - result=$? - set -e - if [[ $result != 0 ]] + if ! output=$(echo '' | "$SOLC" --ast-compact-json - 2>/dev/null) then printError "Incorrect response to --ast-compact-json option with empty stdin" exit 1 @@ -479,8 +470,7 @@ printTask "Testing AST import..." SOLTMPDIR=$(mktemp -d) ( cd "$SOLTMPDIR" - "$REPO_ROOT/scripts/ASTImportTest.sh" - if [ $? -ne 0 ] + if ! "$REPO_ROOT/scripts/ASTImportTest.sh" then rm -rf "$SOLTMPDIR" exit 1 From cf94c3f7df01c51ba5e65ece6280ab90cc9bfd4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20=C5=9Aliwak?= Date: Tue, 2 Feb 2021 15:03:30 +0100 Subject: [PATCH 16/16] Remove scripts that now have no warnings from shellcheck ignore list --- scripts/chk_shellscripts/ignore.txt | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/chk_shellscripts/ignore.txt b/scripts/chk_shellscripts/ignore.txt index 24bdf69c5660..21b60ee2dc16 100644 --- a/scripts/chk_shellscripts/ignore.txt +++ b/scripts/chk_shellscripts/ignore.txt @@ -1,20 +1,10 @@ ./test/cmdlineTests.sh ./scripts/wasm-rebuild/docker-scripts/rebuild_tags.sh -./scripts/wasm-rebuild/docker-scripts/rebuild_current.sh -./scripts/wasm-rebuild/docker-scripts/genbytecode.sh -./scripts/ci/build_emscripten.sh ./scripts/docs_version_pragma_check.sh -./scripts/uniqueErrors.sh ./scripts/tests.sh ./scripts/bytecodecompare/storebytecode.sh ./scripts/deps-ppa/static_z3.sh ./scripts/ASTImportTest.sh -./scripts/install_static_z3.sh ./scripts/install_deps.sh -./scripts/common_cmdline.sh -./scripts/docker_deploy_manual.sh -./scripts/endToEndExtraction/create_traces.sh ./scripts/release_ppa.sh -./scripts/create_source_tarball.sh ./.circleci/soltest.sh -./.circleci/soltest_all.sh