From c07840d60be17a671deb5bf3ec51bc3329a281df Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Wed, 13 Apr 2022 23:09:48 +0200 Subject: [PATCH 01/18] Add `check_prerequisites` function --- .github/workflows/test.yml | 2 ++ src/rsync_offsite_backup.sh | 29 ++++++++++++++--------------- test/test_check_prerequisites.bats | 23 +++++++++++++++++++++++ test/test_helper/common-setup.bash | 19 +++++++++++++++++++ test/true.bats | 5 ----- 5 files changed, 58 insertions(+), 20 deletions(-) create mode 100644 test/test_check_prerequisites.bats create mode 100644 test/test_helper/common-setup.bash delete mode 100644 test/true.bats diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abd1e63..92c464d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,8 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v2 + with: + submodules: "true" - uses: docker://bats/bats:latest with: args: test/ diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index f53bc35..79e42f7 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -18,7 +18,6 @@ CONFIG_FILE="" DEFAULT_INFO="progress2" JOB_NAME="${_PREFIX}${PID}" RSYNC_OPTIONS=( - "$(which --skip-alias --skip-functions rsync)" #--8-bit-output # leave high-bit chars unescaped in output #--acls # preserve ACLs (implies --perms) #--address=ADDRESS # bind address for outgoing socket to daemon @@ -146,20 +145,20 @@ RSYNC_OPTIONS=( #--xattrs # preserve extended attributes ) +function check_prerequisites() { + local required_commands=(dasel jq rsync screen) -# -# Prerequisites -# -for command_name in dasel jq rsync screen -do - # shellcheck disable=SC2248,SC2250 - if ! which $command_name 1>/dev/null 2>&1 - then - echo -n "ERROR: ${command_name} is not available or not in your PATH. " >&2 - echo "Please install ${command_name} and try again." >&2 - exit 1 - fi -done + for command_name in "${required_commands[@]}"; do + # shellcheck disable=SC2248 + if ! which ${command_name} 1>/dev/null 2>&1; then + echo -n "ERROR: ${command_name} is not available or not in your PATH. " >&2 + echo "Please install ${command_name} and try again." >&2 + exit 1 + fi + done +} + +return 0 # FIXME: Remove # @@ -364,5 +363,5 @@ fi # Execute rsync # echo -n 'Starting rsync... ' -screen -dmU -S "${JOB_NAME}" -t "${JOB_NAME}" "${RSYNC_OPTIONS[@]}" +screen -dmU -S "${JOB_NAME}" -t "${JOB_NAME}" "$(which --skip-alias --skip-functions rsync)" "${RSYNC_OPTIONS[@]}" echo 'Done.' diff --git a/test/test_check_prerequisites.bats b/test/test_check_prerequisites.bats new file mode 100644 index 0000000..8785230 --- /dev/null +++ b/test/test_check_prerequisites.bats @@ -0,0 +1,23 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup +} + +teardown() { + PATH="${PATH_BACKUP}" +} + +@test "check_prerequisites should not find dasel executable" { + # GIVEN + # shellcheck disable=SC2123 + PATH='.' + + # WHEN + run check_prerequisites + + # THEN + assert_failure + assert_output 'ERROR: dasel is not available or not in your PATH. Please install dasel and try again.' +} diff --git a/test/test_helper/common-setup.bash b/test/test_helper/common-setup.bash new file mode 100644 index 0000000..d2fdb0e --- /dev/null +++ b/test/test_helper/common-setup.bash @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +_common_setup() { + load 'test_helper/bats-support/load' + load 'test_helper/bats-assert/load' + + # get the containing directory of this file + # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, + # as those will point to the bats executable's location or the preprocessed file respectively + + # shellcheck disable=SC2154 + local project_root="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." >/dev/null 2>&1 && pwd)" + PATH="${project_root}/src:${PATH}" + + # shellcheck disable=SC2034 + PATH_BACKUP="$PATH" + + source rsync_offsite_backup.sh +} diff --git a/test/true.bats b/test/true.bats deleted file mode 100644 index 734547b..0000000 --- a/test/true.bats +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bats - -@test "true" { - true -} From 06e5f7eb559bbfa0009528f3ce83a950f4d7d65c Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Wed, 13 Apr 2022 23:44:48 +0200 Subject: [PATCH 02/18] Add code coverage measuring --- .github/workflows/test.yml | 27 +++++++++++++++++++++++++-- .gitignore | 2 +- README.md | 1 + test/test_check_prerequisites.bats | 2 +- 4 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92c464d..f73697d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,5 @@ --- +# https://github.com/particleflux/kcov-bats-circleci-codeclimate name: Test @@ -19,6 +20,28 @@ jobs: - uses: actions/checkout@v2 with: submodules: "true" - - uses: docker://bats/bats:latest + + - name: Run tests + uses: docker://kcov/kcov:latest + with: + args: kcov --include-path src/ codecov/ ./test/bats/bin/bats test/ + + # https://github.com/particleflux/kcov-bats-circleci-codeclimate/blob/master/.circleci/config.yml + - name: Prepare coverage report + run: |2 + _xml="$(ls -1 codecov/bats*/cobertura.xml | head -1)" + sed -r \ + 's#"bats"#"src"#;s#/github/workspace/##;s#(.+)/#/github/workspace/#' \ + "$_xml" > coverage.xml + - name: Upload coverage report + uses: codecov/codecov-action@v2 + # https://github.com/SimonKagstrom/kcov/blob/master/doc/codecov.md + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: coverage.xml + fail_ci_if_error: false + + - uses: actions/upload-artifact@v3 with: - args: test/ + name: code-coverage-report + path: coverage.xml diff --git a/.gitignore b/.gitignore index 50ac897..06602a1 100644 --- a/.gitignore +++ b/.gitignore @@ -96,4 +96,4 @@ modules.xml # End of https://www.toptal.com/developers/gitignore/api/jetbrains+all -.git \ No newline at end of file +codecov/ diff --git a/README.md b/README.md index 51c89b9..694c857 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ [![Lint](https://github.com/danie1k/rsync-backup/actions/workflows/lint.yml/badge.svg)](https://github.com/danie1k/rsync-backup/actions/workflows/lint.yml) [![Test](https://github.com/danie1k/rsync-backup/actions/workflows/test.yml/badge.svg)](https://github.com/danie1k/rsync-backup/actions/workflows/test.yml) +[![Code Coverage](https://codecov.io/gh/danie1k/rsync-backup/branch/master/graph/badge.svg?token=07IXMZ0DWO)](https://codecov.io/gh/danie1k/rsync-backup) [![MIT License](https://img.shields.io/github/license/danie1k/rsync-backup)](https://github.com/danie1k/rsync-backup/blob/master/LICENSE) # rsync-backup diff --git a/test/test_check_prerequisites.bats b/test/test_check_prerequisites.bats index 8785230..99bcb82 100644 --- a/test/test_check_prerequisites.bats +++ b/test/test_check_prerequisites.bats @@ -6,7 +6,7 @@ setup() { } teardown() { - PATH="${PATH_BACKUP}" + PATH="${PATH_BACKUP:?}" } @test "check_prerequisites should not find dasel executable" { From b5d052a4ddf058f10302e97cc2b45b235b6bdf9f Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 12:38:12 +0200 Subject: [PATCH 03/18] Update `print_usage` function --- src/rsync_offsite_backup.sh | 15 +++++++-------- test/test_print_usage.bats | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 8 deletions(-) create mode 100644 test/test_print_usage.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 79e42f7..de9aec5 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -158,25 +158,24 @@ function check_prerequisites() { done } -return 0 # FIXME: Remove - -# -# Process command line args -# -usage() { +function print_usage() { echo "Usage: ${SELF} [OPTIONS]" echo echo "Options:" - echo " -c string Path to config file for this job" + echo " -c FILE Path to config file for this job" echo " -d Dry run" echo " -l Show list of the currently running jobs and exit" - echo " -n string Custom name of the job" + echo " -n NAME Custom name of the job" echo " -h Shows this help" echo echo "Version ${_VERSION}" } + +return 0 # FIXME: Remove + + [ $# -eq 0 ] && usage && exit 1 while getopts ":c:dhln:" opt diff --git a/test/test_print_usage.bats b/test/test_print_usage.bats new file mode 100644 index 0000000..b964db0 --- /dev/null +++ b/test/test_print_usage.bats @@ -0,0 +1,17 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup +} + +@test "print_usage result" { + # WHEN + run print_usage + + # THEN + assert_success + assert_output --partial 'Usage:' + assert_output --partial 'Options:' + assert_output --partial 'Version' +} From 5b410e9e3ceb8fac0a30ac21a7a3f452fe840b71 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 14:00:03 +0200 Subject: [PATCH 04/18] Add _`error` & `_out` helpers --- src/rsync_offsite_backup.sh | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index de9aec5..6e562aa 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -7,6 +7,8 @@ readonly PID=$$ readonly _PREFIX='rsync:' readonly _VERSION='2022.02' +_error() { echo "ERROR: ${*}" >&2; } +_out() { echo "${*}"; } # # Defaults @@ -151,8 +153,8 @@ function check_prerequisites() { for command_name in "${required_commands[@]}"; do # shellcheck disable=SC2248 if ! which ${command_name} 1>/dev/null 2>&1; then - echo -n "ERROR: ${command_name} is not available or not in your PATH. " >&2 - echo "Please install ${command_name} and try again." >&2 + _error "${command_name} is not available or not in your PATH." \ + "Please install ${command_name} and try again." exit 1 fi done @@ -160,16 +162,16 @@ function check_prerequisites() { function print_usage() { - echo "Usage: ${SELF} [OPTIONS]" - echo - echo "Options:" - echo " -c FILE Path to config file for this job" - echo " -d Dry run" - echo " -l Show list of the currently running jobs and exit" - echo " -n NAME Custom name of the job" - echo " -h Shows this help" - echo - echo "Version ${_VERSION}" + _out "Usage: ${SELF} [OPTIONS]" + _out + _out "Options:" + _out " -c FILE Path to config file for this job" + _out " -d Dry run" + _out " -l Show list of the currently running jobs and exit" + _out " -n NAME Custom name of the job" + _out " -h Shows this help" + _out + _out "Version ${_VERSION}" } From b246b494b3cb9aa2c4e91f4795e62370894ded8e Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 14:02:34 +0200 Subject: [PATCH 05/18] Add `collect_options` function --- src/rsync_offsite_backup.sh | 74 ++++++++++++++++++------------- test/test_collect_options.bats | 79 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 31 deletions(-) create mode 100644 test/test_collect_options.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 6e562aa..9af76d4 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -13,12 +13,7 @@ _out() { echo "${*}"; } # # Defaults # -DRY_RUN_FLAG=0 -LIST_ONLY_FLAG=0 - -CONFIG_FILE="" -DEFAULT_INFO="progress2" -JOB_NAME="${_PREFIX}${PID}" +DEFAULT_INFO='progress2' RSYNC_OPTIONS=( #--8-bit-output # leave high-bit chars unescaped in output #--acls # preserve ACLs (implies --perms) @@ -174,34 +169,51 @@ function print_usage() { _out "Version ${_VERSION}" } +function collect_options() { + if [[ $# -eq 0 ]]; then + print_usage + exit 1 + fi -return 0 # FIXME: Remove - - -[ $# -eq 0 ] && usage && exit 1 + # Values + CONFIG_FILE='' + JOB_NAME="${_PREFIX}${PID}" + # Flags + DRY_RUN_FLAG=0 + LIST_ONLY_FLAG=0 + + while getopts ':c:dhln:-' opt; do + case "${opt}" in + '-') + _error 'Long options are not supported' + exit 2 + ;; + c) + CONFIG_FILE="${OPTARG}" + ;; + d) + DRY_RUN_FLAG=1 + ;; + l) + LIST_ONLY_FLAG=1 + ;; + n) + JOB_NAME="${_PREFIX}${OPTARG}" + ;; + h | *) + print_usage + exit 1 + ;; + esac + done -while getopts ":c:dhln:" opt -do - case "${opt}" in - c) - readonly CONFIG_FILE="${OPTARG}" - ;; - d) - readonly DRY_RUN_FLAG=1 - ;; - l) - readonly LIST_ONLY_FLAG=1 - ;; - n) - readonly JOB_NAME="${_PREFIX}${OPTARG}" - ;; - h|*) - usage - exit 1 - ;; - esac -done + export CONFIG_FILE + export DRY_RUN_FLAG + export JOB_NAME + export LIST_ONLY_FLAG +} +return 0 # FIXME: Remove # # Get/show current jobs diff --git a/test/test_collect_options.bats b/test/test_collect_options.bats new file mode 100644 index 0000000..36e63dc --- /dev/null +++ b/test/test_collect_options.bats @@ -0,0 +1,79 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup + unset CONFIG_FILE + unset DRY_RUN_FLAG + unset JOB_NAME + unset LIST_ONLY_FLAG +} + +@test "collect_options should print usage when no args given" { + # WHEN + run collect_options + + # THEN + assert_failure + assert_output --partial 'Usage:' +} + +@test "collect_options should print usage when '-h' arg given" { + # WHEN + run collect_options -h + + # THEN + assert_failure + assert_output --partial 'Usage:' +} + +@test "collect_options should exit with error when '--' arg given" { + # WHEN + run collect_options --help + + # THEN + assert_failure + assert_output --partial 'Long options are not supported' +} + +@test "collect_options should correctly export 'CONFIG_FILE' variable" { + # GIVEN + local given_file_path='foo/bar.yml' + + # WHEN + collect_options -c "${given_file_path}" + + # THEN + assert_equal $? 0 + assert_equal "${CONFIG_FILE:?}" "${given_file_path}" +} + +@test "collect_options should correctly export 'DRY_RUN_FLAG' variable" { + # WHEN + collect_options -d + + # THEN + assert_equal $? 0 + assert_equal ${DRY_RUN_FLAG:?} 1 +} + +@test "collect_options should correctly export 'JOB_NAME' variable" { + # GIVEN + local given_name='lorem-ipsum' + + # WHEN + collect_options -n "${given_name}" + + # THEN + assert_equal $? 0 + assert_equal "${JOB_NAME:?}" "rsync:${given_name}" +} + +@test "collect_options should correctly export 'LIST_ONLY_FLAG' variable" { + # WHEN + collect_options -l + + # THEN + assert_equal $? 0 + assert_equal ${LIST_ONLY_FLAG:?} 1 +} From 53ffbd2d7158da94471ebaa7a5874e3a119d194a Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 16:42:38 +0200 Subject: [PATCH 06/18] Improve/fix `Makefile` & Lint Shellcheck GitHub Action --- .github/workflows/lint.yml | 12 +++++++----- Makefile | 8 +++++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 56bb941..5e44e28 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,12 +17,14 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v2 - - uses: ludeeus/action-shellcheck@master - env: - SHELLCHECK_OPTS: --enable=all --shell=bash + - run: | + echo -n 'exec find . -maxdepth 2 -name .git -type d -prune -o -type f ' > ci_shellcheck.sh + echo -n '\( -name \*.sh -or -name \*.bats -or -name \*.bash \) -print0 ' >> ci_shellcheck.sh + echo -n '| xargs -0 -r -n1 shellcheck ' >> ci_shellcheck.sh + echo -n '--enable=all --severity=warning --color=never --format gcc' >> ci_shellcheck.sh + - uses: docker://pipelinecomponents/shellcheck:latest with: - format: gcc - severity: warning + args: /bin/sh ci_shellcheck.sh yamllint: name: Yamllint diff --git a/Makefile b/Makefile index dcbcd44..b1d7900 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,8 @@ +bats: + @./test/bats/bin/bats test/ + +bats-cov: + @docker run -it --rm --workdir /github/workspace -v "$PWD":/github/workspace kcov/kcov:latest kcov --include-path src/ codecov/ ./test/bats/bin/bats test/ + shellcheck: - @find . -maxdepth 2 -name .git -type d -prune -o -type f \( -name \*.sh -or -name \*.bats \) -print0 | xargs -0 -r -n1 shellcheck --enable=all --severity=warning --shell=bash --color=always + @find . -maxdepth 2 -name .git -type d -prune -o -type f \( -name \*.sh -or -name \*.bats -or -name \*.bash \) -print0 | xargs -0 -r -n1 shellcheck --enable=all --severity=warning --color=always From 7c43e9bd5d3deee261912d09acbee40853a1706c Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 19:37:18 +0200 Subject: [PATCH 07/18] Add `get_active_jobs` function --- .gitmodules | 3 +++ src/rsync_offsite_backup.sh | 11 ++++---- test/test_check_prerequisites.bats | 1 + test/test_get_active_jobs.bats | 43 ++++++++++++++++++++++++++++++ test/test_helper/mocks | 1 + 5 files changed, 53 insertions(+), 6 deletions(-) create mode 100644 test/test_get_active_jobs.bats create mode 160000 test/test_helper/mocks diff --git a/.gitmodules b/.gitmodules index b7efcb4..7eef4e6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "test/test_helper/bats-assert"] path = test/test_helper/bats-assert url = https://github.com/bats-core/bats-assert.git +[submodule "test/test_helper/mocks"] + path = test/test_helper/mocks + url = https://github.com/jasonkarns/bats-mock diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 9af76d4..55d761b 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -213,13 +213,12 @@ function collect_options() { export LIST_ONLY_FLAG } -return 0 # FIXME: Remove +function get_active_jobs() { + screen -wipe 1>/dev/null 2>&1 || true + screen -list | grep 'Detached' | grep "${_PREFIX}" | awk '{print $1}' +} -# -# Get/show current jobs -# -screen -wipe 1>/dev/null 2>&1 || true -readonly ACTIVE_SCREEN_SESSIONS="$(screen -list | grep 'Detached' | grep "${_PREFIX}" | awk '{print $1}')" +return 0 # FIXME: Remove # shellcheck disable=SC2248,SC2250 if [ $LIST_ONLY_FLAG -eq 1 ] diff --git a/test/test_check_prerequisites.bats b/test/test_check_prerequisites.bats index 99bcb82..f8d8d96 100644 --- a/test/test_check_prerequisites.bats +++ b/test/test_check_prerequisites.bats @@ -9,6 +9,7 @@ teardown() { PATH="${PATH_BACKUP:?}" } + @test "check_prerequisites should not find dasel executable" { # GIVEN # shellcheck disable=SC2123 diff --git a/test/test_get_active_jobs.bats b/test/test_get_active_jobs.bats new file mode 100644 index 0000000..e23076f --- /dev/null +++ b/test/test_get_active_jobs.bats @@ -0,0 +1,43 @@ +#!/usr/bin/env bats + +_mock_screen() { + screen_list_stdout_result='There are screens on:\n' + screen_list_stdout_result+=' 4060579.pts-14.foobar (Detached)\n' + screen_list_stdout_result+=' 4055090.rsync:1234 (Detached)\n' + screen_list_stdout_result+=' 3052708.rsync:lorem (Detached)\n' + screen_list_stdout_result+=' 3050193.rsync:ipsum (Detached)\n' + screen_list_stdout_result+='5 Sockets in /run/screens/S-johndoe.\n' + + stub screen \ + "-list : echo $'${screen_list_stdout_result}'" \ + "-list : echo $'${screen_list_stdout_result}'" +} + +setup() { + load 'test_helper/common-setup' + load 'test_helper/mocks/stub' + _common_setup + _mock_screen +} + +@test "get_active_jobs : verify screen mock" { + # WHEN + run screen -list + + # THEN + assert_output --partial 'There are screens on' + assert_output --partial '4060579.pts-14.foobar (Detached)' + assert_output --partial '5 Sockets in /run/screens/S-johndoe' +} + +@test "get_active_jobs result" { + expected_stdout_result=$'4055090.rsync:1234\n' + expected_stdout_result+=$'3052708.rsync:lorem\n' + expected_stdout_result+='3050193.rsync:ipsum' + + # WHEN + run get_active_jobs + + # THEN + assert_output "${expected_stdout_result}" +} diff --git a/test/test_helper/mocks b/test/test_helper/mocks new file mode 160000 index 0000000..7e0fbf6 --- /dev/null +++ b/test/test_helper/mocks @@ -0,0 +1 @@ +Subproject commit 7e0fbf6bc705bd1b09daa2d5ff88962ddbe832f6 From ed804ed28abf75de2252efa46f5a08fcb1e29154 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 20:41:05 +0200 Subject: [PATCH 08/18] Add `_header` helper --- src/rsync_offsite_backup.sh | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 55d761b..53c74bb 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -5,11 +5,58 @@ shopt -s failglob readonly SELF="${0}" readonly PID=$$ readonly _PREFIX='rsync:' +readonly _TERMWIDTH=78 readonly _VERSION='2022.02' _error() { echo "ERROR: ${*}" >&2; } _out() { echo "${*}"; } +_header() { + local _pad="$(printf '%0.1s' " "{1..500})" + local _seq=$(seq 1 ${_TERMWIDTH}) + + _top() { + echo -n '┌' + printf '─%.0s' ${_seq} + echo '┐' + } + _mid() { + echo -n '├' + printf '─%.0s' ${_seq} + echo '┤' + } + _btm() { + echo -n '└' + printf '─%.0s' ${_seq} + echo '┘' + } + + _top + + # https://unix.stackexchange.com/a/267730 + printf '│%*.*s%s%*.*s│\n' \ + 0 "$(((_TERMWIDTH - ${#1}) / 2))" "${_pad}" \ + "${1}" \ + 0 "$(((_TERMWIDTH + 1 - ${#1}) / 2))" "${_pad}" + + shift + + if [[ $# -eq 0 ]]; then + _btm + return 0 + else + _mid + fi + + for line in "${@}"; do + printf '│ %s %*.*s│\n' \ + "${line}" \ + 0 "$(((_TERMWIDTH - 2 - ${#line})))" "${_pad}" + done + + _btm +} + # # Defaults # From b8702046ab0805896fd5e43dcddf6645166e7fc7 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Thu, 14 Apr 2022 20:58:19 +0200 Subject: [PATCH 09/18] Add `list_active_jobs` function --- src/rsync_offsite_backup.sh | 39 +++++++++++++------------------- test/test_list_active_jobs.bats | 40 +++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 23 deletions(-) create mode 100644 test/test_list_active_jobs.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 53c74bb..6081ceb 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -265,33 +265,26 @@ function get_active_jobs() { screen -list | grep 'Detached' | grep "${_PREFIX}" | awk '{print $1}' } -return 0 # FIXME: Remove +function list_active_jobs() { + local jobs_list="$(get_active_jobs)" -# shellcheck disable=SC2248,SC2250 -if [ $LIST_ONLY_FLAG -eq 1 ] -then - if [ -z "${ACTIVE_SCREEN_SESSIONS}" ] - then - echo 'No active rsync jobs' + if [[ -z "${jobs_list}" ]]; then + _out 'No active rsync jobs' exit 0 + else + _header "CURRENTLY ACTIVE RSYNC SESSIONS" \ + 'To attach to the session run: rsync -r ' + + local i=1 + # shellcheck disable=SC2068 + for job_name in ${jobs_list[@]}; do + printf '(%02d) %s\n' ${i} "${job_name}" + i=$((i + 1)) + done fi +} - _printf_line=' ' - _reattach_cmd="rsync -r " - echo "┌────────────────────────────────────────────────────────────────────────────┐" - echo "│ CURRENTLY ACTIVE RSYNC SESSIONS │" - echo "├────────────────────────────────────────────────────────────────────────────┤" - printf "│ To attach to the session run: %s%s │\n" "${_reattach_cmd}" "${_printf_line:${#_reattach_cmd}}" - echo "└────────────────────────────────────────────────────────────────────────────┘" - - # shellcheck disable=SC2068 - for job_name in ${ACTIVE_SCREEN_SESSIONS[@]} - do - echo "${job_name}" - done - exit 0 -fi - +return 0 # FIXME: Remove # # Read config diff --git a/test/test_list_active_jobs.bats b/test/test_list_active_jobs.bats new file mode 100644 index 0000000..1748731 --- /dev/null +++ b/test/test_list_active_jobs.bats @@ -0,0 +1,40 @@ +#!/usr/bin/env bats + +get_active_jobs_stdout=( + '4055090.rsync:1234' + '3052708.rsync:lorem' + '3050193.rsync:ipsum' +) + +setup() { + load 'test_helper/common-setup' + load 'test_helper/mocks/stub' + _common_setup +} + +@test "list_active_jobs should show message when no jobs" { + # MOCK + get_active_jobs() { echo ''; } + + # WHEN + run list_active_jobs + + # THEN + assert_output 'No active rsync jobs' +} + +@test "list_active_jobs should show header and list of the jobs" { + # MOCK + get_active_jobs() { echo -e "${get_active_jobs_stdout[0]}\n${get_active_jobs_stdout[1]}\n${get_active_jobs_stdout[2]}"; } + + # WHEN + run list_active_jobs + + # THEN + assert_output --partial 'CURRENTLY ACTIVE RSYNC SESSIONS' + assert_output --partial 'To attach to the session run: rsync -r' + assert_output --partial "(01) ${get_active_jobs_stdout[0]}" + assert_output --partial "(02) ${get_active_jobs_stdout[1]}" + assert_output --partial "(03) ${get_active_jobs_stdout[2]}" + refute_output --partial '(04)' +} From 3af8e191a9cef7f7452cf45ee3b635224a987c7a Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Fri, 15 Apr 2022 13:14:11 +0200 Subject: [PATCH 10/18] Add `_get_version` helper --- src/rsync_offsite_backup.sh | 34 +++++++++++++++++++++-------- test/test_helpers.bats | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 9 deletions(-) create mode 100644 test/test_helpers.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 6081ceb..d5ef07e 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -1,12 +1,11 @@ #!/bin/bash set -eu -o pipefail shopt -s failglob +readonly __VERSION__='2022.02' -readonly SELF="${0}" -readonly PID=$$ -readonly _PREFIX='rsync:' -readonly _TERMWIDTH=78 -readonly _VERSION='2022.02' +## +## HELPERS +## _error() { echo "ERROR: ${*}" >&2; } _out() { echo "${*}"; } @@ -57,9 +56,26 @@ _header() { _btm } -# -# Defaults -# +_get_version() { + local needle="${2:-NF}" + + if ! raw_version=$("${1}" --version 2>/dev/null); then + echo '-' + return 0 + fi + + echo "${raw_version}" | head -1 | awk "{print \$${needle}}" +} + +## +## DEFAULTS +## + +readonly SELF="${0}" +readonly PID=$$ +readonly _PREFIX='rsync:' +readonly _TERMWIDTH=78 + DEFAULT_INFO='progress2' RSYNC_OPTIONS=( #--8-bit-output # leave high-bit chars unescaped in output @@ -213,7 +229,7 @@ function print_usage() { _out " -n NAME Custom name of the job" _out " -h Shows this help" _out - _out "Version ${_VERSION}" + _out "Version ${__VERSION__}" } function collect_options() { diff --git a/test/test_helpers.bats b/test/test_helpers.bats new file mode 100644 index 0000000..8d4d34c --- /dev/null +++ b/test/test_helpers.bats @@ -0,0 +1,43 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup +} + +teardown() { + unset foo_bar +} + +@test "_get_version : when no needle given" { + # MOCK + foo_bar() { echo 'foo_bar version 42.314'; } + + # WHEN + run _get_version foo_bar + + # THEN + assert_success + assert_output '42.314' +} + +@test "_get_version : when needle given" { + # MOCK + foo_bar() { echo 'foo_bar version 42 3.14'; } + + # WHEN + run _get_version foo_bar 3 + + # THEN + assert_success + assert_output '42' +} + +@test "_get_version : when command/executable does not exist" { + # WHEN + run _get_version inexistent_command_name + + # THEN + assert_success + assert_output --regexp '^-$' +} From e1a4a96edc47a9e3c1854a7f54e86b70ad685ad7 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Fri, 15 Apr 2022 21:41:20 +0200 Subject: [PATCH 11/18] Add `_get_config_*` helpers --- .github/workflows/lint.yml | 7 +- .github/workflows/test.yml | 14 ++- .gitignore | 3 + README.md | 3 +- src/rsync_offsite_backup.sh | 50 +++++++- test/bin/.gitkeep | 0 test/bin/ci_shellcheck.sh | 4 + test/bin/ci_tests.sh | 3 + test/test_check_prerequisites.bats | 6 +- test/test_helper/common-setup.bash | 4 +- test/test_helpers.bats | 188 ++++++++++++++++++++++++++++- 11 files changed, 260 insertions(+), 22 deletions(-) create mode 100644 test/bin/.gitkeep create mode 100755 test/bin/ci_shellcheck.sh create mode 100755 test/bin/ci_tests.sh diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5e44e28..454a54c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -17,14 +17,9 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@v2 - - run: | - echo -n 'exec find . -maxdepth 2 -name .git -type d -prune -o -type f ' > ci_shellcheck.sh - echo -n '\( -name \*.sh -or -name \*.bats -or -name \*.bash \) -print0 ' >> ci_shellcheck.sh - echo -n '| xargs -0 -r -n1 shellcheck ' >> ci_shellcheck.sh - echo -n '--enable=all --severity=warning --color=never --format gcc' >> ci_shellcheck.sh - uses: docker://pipelinecomponents/shellcheck:latest with: - args: /bin/sh ci_shellcheck.sh + args: /bin/sh test/bin/ci_shellcheck.sh yamllint: name: Yamllint diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f73697d..9eff7ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,18 +21,26 @@ jobs: with: submodules: "true" + - name: Download dasel + uses: wei/wget@v1 + with: + args: -O test/bin/dasel https://github.com/TomWright/dasel/releases/download/v1.24.1/dasel_linux_amd64 + - name: Download yq + uses: wei/wget@v1 + with: + args: -O test/bin/yq https://github.com/mikefarah/yq/releases/download/v4.24.5/yq_linux_amd64 + - name: Run tests uses: docker://kcov/kcov:latest with: - args: kcov --include-path src/ codecov/ ./test/bats/bin/bats test/ + args: /bin/sh test/bin/ci_tests.sh # https://github.com/particleflux/kcov-bats-circleci-codeclimate/blob/master/.circleci/config.yml - name: Prepare coverage report run: |2 - _xml="$(ls -1 codecov/bats*/cobertura.xml | head -1)" sed -r \ 's#"bats"#"src"#;s#/github/workspace/##;s#(.+)/#/github/workspace/#' \ - "$_xml" > coverage.xml + "$(ls -1 codecov/bats*/cobertura.xml | head -1)" > coverage.xml - name: Upload coverage report uses: codecov/codecov-action@v2 # https://github.com/SimonKagstrom/kcov/blob/master/doc/codecov.md diff --git a/.gitignore b/.gitignore index 06602a1..9e3452d 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,6 @@ modules.xml # End of https://www.toptal.com/developers/gitignore/api/jetbrains+all codecov/ +test/bin/* +!test/bin/ci_*.sh +!test/bin/.gitkeep diff --git a/README.md b/README.md index 694c857..9de1334 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,7 @@ A simple bash script wrapper for the rsync command ## Requirements - `bash` (or any other bash-compatible shell; `sh`, `dash`, etc. are not supported) -- `dasel` (https://github.com/TomWright/dasel) -- `jq` (https://github.com/stedolan/jq) +- [dasel](https://github.com/TomWright/dasel) or [yq](https://github.com/mikefarah/yq) - `rsync` - `screen` - `envsubst` (optional, a part of `gettext` package) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index d5ef07e..a13dddf 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -1,7 +1,8 @@ #!/bin/bash -set -eu -o pipefail -shopt -s failglob +set -eu +o pipefail + readonly __VERSION__='2022.02' +readonly __NO_VALUE__='-' ## ## HELPERS @@ -60,13 +61,52 @@ _get_version() { local needle="${2:-NF}" if ! raw_version=$("${1}" --version 2>/dev/null); then - echo '-' + echo "${__NO_VALUE__}" return 0 fi echo "${raw_version}" | head -1 | awk "{print \$${needle}}" } +_get_config_list() { + local _result='null' + + if [[ "${DASEL_VER:?}" != "-" ]]; then + # dasel + _result="$( + echo "${1}" | dasel --null --plain -p yaml -m \ + --format '{{ selectMultiple ".[*]" | format "{{ select \".\" }}{{ newline }}" }}' \ + "${2}" 2>/dev/null + )" + elif [[ "${YQ_VER:?}" != "${__NO_VALUE__}" ]]; then + # yq + local _result="$( + echo "${1}" | yq --no-colors e "${2}" - 2>/dev/null \ + | yq --no-colors e $'join("\n")' - 2>/dev/null + )" + fi + + if [[ "${_result}" == null* ]] || [[ -z "${_result}" ]]; then + echo "${__NO_VALUE__}" + else + echo "${_result}" + fi +} + +_get_config_value() { + local _result='null' + + if [[ "${DASEL_VER:?}" != "${__NO_VALUE__}" ]]; then + # dasel + _result="$(echo "${1}" | dasel --null --plain -p yaml -m "${2}")" + elif [[ "${YQ_VER:?}" != "${__NO_VALUE__}" ]]; then + # yq + _result="$(echo "${1}" | yq --no-colors e "${2}" -)" + fi + + [[ "${_result}" == null* ]] && echo "${__NO_VALUE__}" || echo "${_result}" +} + ## ## DEFAULTS ## @@ -206,13 +246,13 @@ RSYNC_OPTIONS=( ) function check_prerequisites() { - local required_commands=(dasel jq rsync screen) + local required_commands=(awk grep head rsync screen) for command_name in "${required_commands[@]}"; do # shellcheck disable=SC2248 if ! which ${command_name} 1>/dev/null 2>&1; then _error "${command_name} is not available or not in your PATH." \ - "Please install ${command_name} and try again." + "Please install ${command_name} and try again." exit 1 fi done diff --git a/test/bin/.gitkeep b/test/bin/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/bin/ci_shellcheck.sh b/test/bin/ci_shellcheck.sh new file mode 100755 index 0000000..86e3c80 --- /dev/null +++ b/test/bin/ci_shellcheck.sh @@ -0,0 +1,4 @@ +exec find . -maxdepth 2 -name .git -type d -prune -o \ + -type f \( -name \*.sh -or -name \*.bats -or -name \*.bash \) -print0 \ + | xargs -0 -r -n1 \ + shellcheck --enable=all --severity=warning --color=never --format gcc diff --git a/test/bin/ci_tests.sh b/test/bin/ci_tests.sh new file mode 100755 index 0000000..0b1db4d --- /dev/null +++ b/test/bin/ci_tests.sh @@ -0,0 +1,3 @@ +set -e +/bin/chmod a+x test/bin/* +exec kcov --include-path src/ codecov/ ./test/bats/bin/bats test/ diff --git a/test/test_check_prerequisites.bats b/test/test_check_prerequisites.bats index f8d8d96..bb20235 100644 --- a/test/test_check_prerequisites.bats +++ b/test/test_check_prerequisites.bats @@ -9,8 +9,7 @@ teardown() { PATH="${PATH_BACKUP:?}" } - -@test "check_prerequisites should not find dasel executable" { +@test "check_prerequisites should not find awk executable" { # GIVEN # shellcheck disable=SC2123 PATH='.' @@ -18,7 +17,8 @@ teardown() { # WHEN run check_prerequisites + PATH="${PATH_BACKUP:?}" # THEN assert_failure - assert_output 'ERROR: dasel is not available or not in your PATH. Please install dasel and try again.' + assert_output 'ERROR: awk is not available or not in your PATH. Please install awk and try again.' } diff --git a/test/test_helper/common-setup.bash b/test/test_helper/common-setup.bash index d2fdb0e..26af4a2 100644 --- a/test/test_helper/common-setup.bash +++ b/test/test_helper/common-setup.bash @@ -10,10 +10,10 @@ _common_setup() { # shellcheck disable=SC2154 local project_root="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." >/dev/null 2>&1 && pwd)" - PATH="${project_root}/src:${PATH}" + PATH="${project_root}/src:${project_root}/test/bin:${PATH}" # shellcheck disable=SC2034 - PATH_BACKUP="$PATH" + PATH_BACKUP="${PATH}" source rsync_offsite_backup.sh } diff --git a/test/test_helpers.bats b/test/test_helpers.bats index 8d4d34c..23f0ca4 100644 --- a/test/test_helpers.bats +++ b/test/test_helpers.bats @@ -1,15 +1,33 @@ #!/usr/bin/env bats +# shellcheck disable=SC2034 + +given_yaml_config=' +path: + source: /tmp/ + remote: tmp/ +rsync: + info: + - progress2 + - name0 +' setup() { load 'test_helper/common-setup' _common_setup + + if [[ "${BATS_TEST_NUMBER:?}" -eq 1 ]]; then + echo "DEBUG: $(dasel --version)" >&3 + echo "DEBUG: $(yq --version)" >&3 + fi } teardown() { unset foo_bar } -@test "_get_version : when no needle given" { +# _get_version ----------------------------------------------------------------- + +@test "_get_version : when no needle given" { # MOCK foo_bar() { echo 'foo_bar version 42.314'; } @@ -41,3 +59,171 @@ teardown() { assert_success assert_output --regexp '^-$' } + +# _get_config_list ------------------------------------------------------------- + +# dasel + +@test "_get_config_list : dasel : when a valid path given" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + run _get_config_list "${given_yaml_config}" '.rsync.info' + + # THEN + assert_success + assert_output $'progress2\nname0' +} + +@test "_get_config_list : dasel : when path to non-list given" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + result="$(_get_config_list "${given_yaml_config}" '.path.source')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} + +@test "_get_config_list : dasel : when invalid path given" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + result="$(_get_config_list "${given_yaml_config}" '.foo.bar')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} + +# yq + +@test "_get_config_list : yq : when a valid path given" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + run _get_config_list "${given_yaml_config}" '.rsync.info' + + # THEN + assert_success + assert_output $'progress2\nname0' +} + +@test "_get_config_list : yq : when path to non-list given" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + result="$(_get_config_list "${given_yaml_config}" '.path.source')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} + +@test "_get_config_list : yq : when invalid path given" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + result="$(_get_config_list "${given_yaml_config}" '.foo.bar')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} + +# _get_config_value ------------------------------------------------------------- + +# dasel + +@test "_get_config_value : dasel : when a valid path given" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + run _get_config_value "${given_yaml_config}" '.path.source' + + # THEN + assert_success + assert_output '/tmp/' +} + +@test "_get_config_value : dasel : when path to non-plain-value" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + result="$(_get_config_value "${given_yaml_config}" '.rsync.info')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" '[progress2 name0]' +} + +@test "_get_config_value : dasel : when invalid path given" { + # MOCK + DASEL_VER='anything' + YQ_VER="${__NO_VALUE__:?}" + + # WHEN + result="$(_get_config_value "${given_yaml_config}" '.foo.bar')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} + +# yq + +@test "_get_config_value : yq : when a valid path given" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + run _get_config_value "${given_yaml_config}" '.path.source' + + # THEN + assert_success + assert_output '/tmp/' +} + +@test "_get_config_value : yq : when path to non-plain-value" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + result="$(_get_config_value "${given_yaml_config}" '.rsync.info')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" $'- progress2\n- name0' +} + +@test "_get_config_value : yq : when invalid path given" { + # MOCK + DASEL_VER="${__NO_VALUE__:?}" + YQ_VER='anything' + + # WHEN + result="$(_get_config_value "${given_yaml_config}" '.foo.bar')" + + # THEN + assert_equal $? 0 + assert_equal "${result}" "${__NO_VALUE__:?}" +} From 13d0ae86b78574f18b2ebf21f1a9d2940fc5df8e Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Sat, 16 Apr 2022 21:19:37 +0200 Subject: [PATCH 12/18] Add `get_3rd_parties_versions` & update `print_usage` --- src/rsync_offsite_backup.sh | 25 +++++++++++++++++--- test/test_collect_options.bats | 16 +++++++++++++ test/test_get_3rd_parties_versions.bats | 31 +++++++++++++++++++++++++ test/test_print_usage.bats | 20 +++++++++++++++- 4 files changed, 88 insertions(+), 4 deletions(-) create mode 100644 test/test_get_3rd_parties_versions.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index a13dddf..0611e6d 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -258,9 +258,15 @@ function check_prerequisites() { done } - function print_usage() { - _out "Usage: ${SELF} [OPTIONS]" + _header "$(basename "${SELF}") v${__VERSION__}" \ + "Usage: ${SELF} [OPTIONS]" \ + '' \ + "dasel: ${DASEL_VER:?}" \ + "envsubst: ${ENVSUBST_VER:?}" \ + "rsync: ${RSYNC_VER:?} (protocol version ${RSYNC_PROTOCOL_VER:?})" \ + "yq: ${YQ_VER:?}" + _out _out "Options:" _out " -c FILE Path to config file for this job" @@ -269,7 +275,6 @@ function print_usage() { _out " -n NAME Custom name of the job" _out " -h Shows this help" _out - _out "Version ${__VERSION__}" } function collect_options() { @@ -340,6 +345,20 @@ function list_active_jobs() { fi } +function get_3rd_parties_versions() { + DASEL_VER="$(_get_version dasel)" + ENVSUBST_VER="$(_get_version envsubst)" + RSYNC_PROTOCOL_VER="$(_get_version rsync)" + RSYNC_VER="$(_get_version rsync 3)" + YQ_VER="$(_get_version yq)" + + export DASEL_VER + export ENVSUBST_VER + export RSYNC_PROTOCOL_VER + export RSYNC_VER + export YQ_VER +} + return 0 # FIXME: Remove # diff --git a/test/test_collect_options.bats b/test/test_collect_options.bats index 36e63dc..8caa6f7 100644 --- a/test/test_collect_options.bats +++ b/test/test_collect_options.bats @@ -3,10 +3,26 @@ setup() { load 'test_helper/common-setup' _common_setup + unset CONFIG_FILE unset DRY_RUN_FLAG unset JOB_NAME unset LIST_ONLY_FLAG + + # MOCK + export DASEL_VER='42' + export ENVSUBST_VER='42' + export RSYNC_PROTOCOL_VER='42' + export RSYNC_VER='42' + export YQ_VER='42' +} + +teardown() { + unset DASEL_VER + unset ENVSUBST_VER + unset RSYNC_PROTOCOL_VER + unset RSYNC_VER + unset YQ_VER } @test "collect_options should print usage when no args given" { diff --git a/test/test_get_3rd_parties_versions.bats b/test/test_get_3rd_parties_versions.bats new file mode 100644 index 0000000..eeb0712 --- /dev/null +++ b/test/test_get_3rd_parties_versions.bats @@ -0,0 +1,31 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup + + unset DASEL_VER + unset ENVSUBST_VER + unset RSYNC_PROTOCOL_VER + unset RSYNC_VER + unset YQ_VER +} + +@test "get_3rd_parties_versions" { + # MOCK + dasel() { echo 'lorem ipsum 42'; } + envsubst() { echo 'lorem ipsum 42'; } + rsync() { echo 'lorem ipsum foo bar 42'; } + yq() { echo 'lorem ipsum 42'; } + + # WHEN + get_3rd_parties_versions + + # THEN + assert_equal $? 0 + assert_equal "${DASEL_VER:?}" '42' + assert_equal "${ENVSUBST_VER:?}" '42' + assert_equal "${RSYNC_PROTOCOL_VER:?}" '42' + assert_equal "${RSYNC_VER:?}" 'foo' + assert_equal "${YQ_VER:?}" '42' +} diff --git a/test/test_print_usage.bats b/test/test_print_usage.bats index b964db0..7214b89 100644 --- a/test/test_print_usage.bats +++ b/test/test_print_usage.bats @@ -3,6 +3,21 @@ setup() { load 'test_helper/common-setup' _common_setup + + # MOCK + export DASEL_VER='42' + export ENVSUBST_VER='42' + export RSYNC_PROTOCOL_VER='42' + export RSYNC_VER='42' + export YQ_VER='42' +} + +teardown() { + unset DASEL_VER + unset ENVSUBST_VER + unset RSYNC_PROTOCOL_VER + unset RSYNC_VER + unset YQ_VER } @test "print_usage result" { @@ -13,5 +28,8 @@ setup() { assert_success assert_output --partial 'Usage:' assert_output --partial 'Options:' - assert_output --partial 'Version' + assert_output --partial 'dasel:' + assert_output --partial 'envsubst:' + assert_output --partial 'rsync:' + assert_output --partial 'yq:' } From d2f28c78a0879622c2ceb46915eadcaf572825c3 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Sat, 16 Apr 2022 22:15:26 +0200 Subject: [PATCH 13/18] Add `get_config_file` function --- src/rsync_offsite_backup.sh | 27 +++++++++-------- test/test_config.bats | 59 +++++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 test/test_config.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 0611e6d..26b757b 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -359,24 +359,25 @@ function get_3rd_parties_versions() { export YQ_VER } +function get_config_file() { + if [[ ! -r "${CONFIG_FILE}" ]]; then + _error "Given config file path is invalid: '${CONFIG_FILE}'" + exit 1 + fi + + if which envsubst 1>/dev/null 2>&1; then + envsubst <"${CONFIG_FILE}" + else + cat "${CONFIG_FILE}" + fi +} + return 0 # FIXME: Remove # # Read config # -if [[ ! -r "${CONFIG_FILE}" ]] -then - echo "ERROR: Given config file path is invalid: '${CONFIG_FILE}'" >&2 - exit 1 -fi - -if which envsubst 1>/dev/null 2>&1 -then - readonly _CONF="$(envsubst < "${CONFIG_FILE}")" -else - readonly _CONF="$(cat "${CONFIG_FILE}")" -fi - +_CONF='' readonly SSH_KEY="$(echo "${_CONF}" | dasel --null -c '.ssh.key' -p yaml 2>/dev/null)" readonly SSH_HOST="$(echo "${_CONF}" | dasel --null -c '.ssh.host' -p yaml 2>/dev/null)" readonly SSH_PORT="$(echo "${_CONF}" | dasel --null -c '.ssh.port' -p yaml 2>/dev/null)" diff --git a/test/test_config.bats b/test/test_config.bats new file mode 100644 index 0000000..dce4dd3 --- /dev/null +++ b/test/test_config.bats @@ -0,0 +1,59 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup + + export TMPFILE="$(mktemp)" +} + +teardown() { + rm -f "${TMPFILE}" + unset CONFIG_FILE + unset TMPFILE +} + +# get_config_file -------------------------------------------------------------- + +@test "get_config_file should fail with error message for unreadable path" { + # GIVEN + export CONFIG_FILE='./inexistent/path' + + # WHEN + run get_config_file + + # THEN + assert_failure + assert_output --partial "Given config file path is invalid: '${CONFIG_FILE}'" +} + +@test "get_config_file should output config file contents if path is valid" { + # GIVEN + given_config_contents=$'foo bar\nlorem ipsum' + echo "${given_config_contents}" >"${TMPFILE}" + export CONFIG_FILE="${TMPFILE}" + + # WHEN + run get_config_file + + # THEN + assert_success + assert_output "${given_config_contents}" +} + +@test "get_config_file should output config file contents through envsubst" { + # GIVEN + given_config_contents=$'foo bar\nlorem ipsum' + echo "${given_config_contents}" >"${TMPFILE}" + export CONFIG_FILE="${TMPFILE}" + + # MOCK + envsubst() { echo "lorem ipsum config:$( Date: Sat, 16 Apr 2022 22:33:08 +0200 Subject: [PATCH 14/18] Add `load_config` function --- src/rsync_offsite_backup.sh | 48 ++++++++++--------- test/test_config.bats | 77 +++++++++++++++++++++++++++++- test/test_helper/common-setup.bash | 8 ++-- test/test_list_active_jobs.bats | 1 - 4 files changed, 104 insertions(+), 30 deletions(-) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 26b757b..59dbd82 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -372,29 +372,33 @@ function get_config_file() { fi } -return 0 # FIXME: Remove +function load_config() { + local conf="$(get_config_file)" -# -# Read config -# -_CONF='' -readonly SSH_KEY="$(echo "${_CONF}" | dasel --null -c '.ssh.key' -p yaml 2>/dev/null)" -readonly SSH_HOST="$(echo "${_CONF}" | dasel --null -c '.ssh.host' -p yaml 2>/dev/null)" -readonly SSH_PORT="$(echo "${_CONF}" | dasel --null -c '.ssh.port' -p yaml 2>/dev/null)" -readonly SSH_USER="$(echo "${_CONF}" | dasel --null -c '.ssh.user' -p yaml 2>/dev/null)" - -readonly PATH_SOURCE="$(echo "${_CONF}" | dasel --null -c '.path.source' -p yaml 2>/dev/null)" -readonly PATH_REMOTE="$(echo "${_CONF}" | dasel --null -c '.path.remote' -p yaml 2>/dev/null)" - -for var_name in SSH_KEY SSH_HOST SSH_PORT SSH_USER PATH_SOURCE PATH_REMOTE -do - if [[ "${!var_name}" == '' ]] - then - echo "ERROR: Config file syntax is invalid." >&2 - exit 1 - fi -done + SSH_KEY="$(_get_config_value "${conf}" '.ssh.key')" + SSH_HOST="$(_get_config_value "${conf}" '.ssh.host')" + SSH_PORT="$(_get_config_value "${conf}" '.ssh.port')" + SSH_USER="$(_get_config_value "${conf}" '.ssh.user')" + + PATH_SOURCE="$(_get_config_value "${conf}" '.path.source')" + PATH_REMOTE="$(_get_config_value "${conf}" '.path.remote')" + for var_name in SSH_KEY SSH_HOST SSH_PORT SSH_USER PATH_SOURCE PATH_REMOTE; do + if [[ "${!var_name}" == '' ]]; then + _error 'Config file syntax is invalid.' + exit 1 + fi + done + + export SSH_KEY + export SSH_HOST + export SSH_PORT + export SSH_USER + export PATH_SOURCE + export PATH_REMOTE +} + +return 0 # FIXME: Remove # # Parse ssh connection parameters @@ -408,7 +412,7 @@ RSYNC_OPTIONS+=( # # Parse info flags # -readonly info_json="$(echo "${_CONF}" | dasel -c -p yaml -r yaml -w json '.rsync.info' 2>/dev/null || echo "[\"${DEFAULT_INFO}\"]")" +readonly info_json="$(echo "${_CONF:-}" | dasel -c -p yaml -r yaml -w json '.rsync.info' 2>/dev/null || echo "[\"${DEFAULT_INFO}\"]")" RSYNC_OPTIONS+=( --info # fine-grained informational verbosity diff --git a/test/test_config.bats b/test/test_config.bats index dce4dd3..e1a6786 100644 --- a/test/test_config.bats +++ b/test/test_config.bats @@ -8,9 +8,21 @@ setup() { } teardown() { - rm -f "${TMPFILE}" + set -e + rm -f "${TMPFILE}" "${PROJECT_ROOT:?}/test/bin/envsubst" + + set +e unset CONFIG_FILE + unset DASEL_VER + unset YQ_VER unset TMPFILE + + unset SSH_KEY + unset SSH_HOST + unset SSH_PORT + unset SSH_USER + unset PATH_SOURCE + unset PATH_REMOTE } # get_config_file -------------------------------------------------------------- @@ -48,7 +60,12 @@ teardown() { export CONFIG_FILE="${TMPFILE}" # MOCK - envsubst() { echo "lorem ipsum config:$( "${PROJECT_ROOT}/test/bin/envsubst" + chmod a+x "${PROJECT_ROOT}/test/bin/envsubst" # WHEN run get_config_file @@ -57,3 +74,59 @@ teardown() { assert_success assert_output "lorem ipsum config:${given_config_contents}" } + +# load_config ------------------------------------------------------------------ + +@test "load_config should load config file using dasel" { + # GIVEN + export DASEL_VER='irrelevant' + export YQ_VER="${__NO_VALUE__:?}" + export CONFIG_FILE="${PROJECT_ROOT:?}/src/example.config.yml" + + assert_equal "${SSH_KEY:-}" '' + assert_equal "${SSH_HOST:-}" '' + assert_equal "${SSH_PORT:-}" '' + assert_equal "${SSH_USER:-}" '' + assert_equal "${PATH_SOURCE:-}" '' + assert_equal "${PATH_REMOTE:-}" '' + + # WHEN + load_config + + # THEN + assert_equal $? 0 + # shellcheck disable=SC2088 + assert_equal "${SSH_KEY}" '~/.ssh/id_rda' + assert_equal "${SSH_HOST}" 'example.com' + assert_equal "${SSH_PORT}" '22' + assert_equal "${SSH_USER}" 'johndoe' + assert_equal "${PATH_SOURCE}" '/tmp/' + assert_equal "${PATH_REMOTE}" 'tmp/' +} + +@test "load_config should load config file using yq" { + # GIVEN + export DASEL_VER="${__NO_VALUE__:?}" + export YQ_VER='irrelevant' + export CONFIG_FILE="${PROJECT_ROOT:?}/src/example.config.yml" + + assert_equal "${SSH_KEY:-}" '' + assert_equal "${SSH_HOST:-}" '' + assert_equal "${SSH_PORT:-}" '' + assert_equal "${SSH_USER:-}" '' + assert_equal "${PATH_SOURCE:-}" '' + assert_equal "${PATH_REMOTE:-}" '' + + # WHEN + load_config + + # THEN + assert_equal $? 0 + # shellcheck disable=SC2088 + assert_equal "${SSH_KEY}" '~/.ssh/id_rda' + assert_equal "${SSH_HOST}" 'example.com' + assert_equal "${SSH_PORT}" '22' + assert_equal "${SSH_USER}" 'johndoe' + assert_equal "${PATH_SOURCE}" '/tmp/' + assert_equal "${PATH_REMOTE}" 'tmp/' +} diff --git a/test/test_helper/common-setup.bash b/test/test_helper/common-setup.bash index 26af4a2..6b89021 100644 --- a/test/test_helper/common-setup.bash +++ b/test/test_helper/common-setup.bash @@ -8,12 +8,10 @@ _common_setup() { # use $BATS_TEST_FILENAME instead of ${BASH_SOURCE[0]} or $0, # as those will point to the bats executable's location or the preprocessed file respectively - # shellcheck disable=SC2154 - local project_root="$(cd "$(dirname "${BATS_TEST_FILENAME}")/.." >/dev/null 2>&1 && pwd)" - PATH="${project_root}/src:${project_root}/test/bin:${PATH}" + export PROJECT_ROOT="$(cd "$(dirname "${BATS_TEST_FILENAME:?}")/.." >/dev/null 2>&1 && pwd)" + PATH="${PROJECT_ROOT}/src:${PROJECT_ROOT}/test/bin:${PATH}" - # shellcheck disable=SC2034 - PATH_BACKUP="${PATH}" + export PATH_BACKUP="${PATH}" source rsync_offsite_backup.sh } diff --git a/test/test_list_active_jobs.bats b/test/test_list_active_jobs.bats index 1748731..0c9b889 100644 --- a/test/test_list_active_jobs.bats +++ b/test/test_list_active_jobs.bats @@ -8,7 +8,6 @@ get_active_jobs_stdout=( setup() { load 'test_helper/common-setup' - load 'test_helper/mocks/stub' _common_setup } From ce15887c6d9256c6420e97a0cd024e3dd675bb17 Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Sun, 17 Apr 2022 12:47:24 +0200 Subject: [PATCH 15/18] Rename `load_config` -> `load_required_config` --- src/rsync_offsite_backup.sh | 2 +- test/test_config.bats | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 59dbd82..c17db2e 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -372,7 +372,7 @@ function get_config_file() { fi } -function load_config() { +function load_required_config() { local conf="$(get_config_file)" SSH_KEY="$(_get_config_value "${conf}" '.ssh.key')" diff --git a/test/test_config.bats b/test/test_config.bats index e1a6786..4956de5 100644 --- a/test/test_config.bats +++ b/test/test_config.bats @@ -75,9 +75,9 @@ teardown() { assert_output "lorem ipsum config:${given_config_contents}" } -# load_config ------------------------------------------------------------------ +# load_required_config --------------------------------------------------------- -@test "load_config should load config file using dasel" { +@test "load_required_config should load config file using dasel" { # GIVEN export DASEL_VER='irrelevant' export YQ_VER="${__NO_VALUE__:?}" @@ -91,7 +91,7 @@ teardown() { assert_equal "${PATH_REMOTE:-}" '' # WHEN - load_config + load_required_config # THEN assert_equal $? 0 @@ -104,7 +104,7 @@ teardown() { assert_equal "${PATH_REMOTE}" 'tmp/' } -@test "load_config should load config file using yq" { +@test "load_required_config should load config file using yq" { # GIVEN export DASEL_VER="${__NO_VALUE__:?}" export YQ_VER='irrelevant' @@ -118,7 +118,7 @@ teardown() { assert_equal "${PATH_REMOTE:-}" '' # WHEN - load_config + load_required_config # THEN assert_equal $? 0 From ce3c5bd98f4e7d1dbfd8e02f7e683833be530bee Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Sun, 17 Apr 2022 12:48:17 +0200 Subject: [PATCH 16/18] Add `load_optional_config` function --- src/rsync_offsite_backup.sh | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index c17db2e..98a5883 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -398,6 +398,14 @@ function load_required_config() { export PATH_REMOTE } +function load_optional_config() { + local conf="$(get_config_file)" + + RSYNC_INFO="$(_get_config_list "${conf}" '.rsync.info')" + + export RSYNC_INFO +} + return 0 # FIXME: Remove # @@ -409,17 +417,6 @@ RSYNC_OPTIONS+=( ) -# -# Parse info flags -# -readonly info_json="$(echo "${_CONF:-}" | dasel -c -p yaml -r yaml -w json '.rsync.info' 2>/dev/null || echo "[\"${DEFAULT_INFO}\"]")" - -RSYNC_OPTIONS+=( - --info # fine-grained informational verbosity - "$(echo "${info_json}" | jq -r '. | join(",")' 2>/dev/null || echo "${DEFAULT_INFO}")" -) - - # # Dry run parameters # From 02257f1c9542995308663edeea286273571bcbfa Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Sun, 17 Apr 2022 12:57:56 +0200 Subject: [PATCH 17/18] Add `collect_rsync_options` function --- src/rsync_offsite_backup.sh | 311 ++++++++++++++------------- test/test_collect_rsync_options.bats | 93 ++++++++ 2 files changed, 255 insertions(+), 149 deletions(-) create mode 100644 test/test_collect_rsync_options.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 98a5883..6d1191c 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -116,134 +116,9 @@ readonly PID=$$ readonly _PREFIX='rsync:' readonly _TERMWIDTH=78 -DEFAULT_INFO='progress2' -RSYNC_OPTIONS=( - #--8-bit-output # leave high-bit chars unescaped in output - #--acls # preserve ACLs (implies --perms) - #--address=ADDRESS # bind address for outgoing socket to daemon - #--append # append data onto shorter files - #--append-verify # --append w/old data in file checksum - #--archive # archive mode; equals: '--devices --group --links --owner --perms --recursive --specials --times' - #--atimes # preserve access (use) times - #--backup # make backups (see --suffix & --backup-dir) - #--backup-dir=DIR # make backups into hierarchy based in DIR - #--block-size=SIZE # force a fixed checksum block-size - #--blocking-io # use blocking I/O for the remote shell - #--bwlimit=RATE # limit socket I/O bandwidth - #--checksum # skip based on checksum, not mod-time & size - #--checksum-choice=STR # choose the checksum algorithm - #--checksum-seed=NUM # set block/file checksum seed (advanced) - #--chmod=CHMOD # affect file and/or directory permissions - #--chown=USER:GROUP # simple username/groupname mapping - #--compare-dest=DIR # also compare destination files relative to DIR - --compress # compress file data during the transfer - #--compress-choice=STR # choose the compression algorithm - #--compress-level=NUM # explicitly set compression level - #--contimeout=SECONDS # set daemon connection timeout in seconds - #--copy-as=USER[:GROUP] # specify user & optional group for the copy - #--copy-dest=DIR # ... and include copies of unchanged files - #--copy-dirlinks # transform symlink to dir into referent dir - #--copy-links # transform symlink into referent file/dir - #--copy-unsafe-links # only "unsafe" symlinks are transformed - #--crtimes # preserve create times (newness) - #--cvs-exclude # auto-ignore files in the same way CVS does - #--debug=FLAGS # fine-grained debug verbosity - #--delay-updates # put all updated files into place at end - --delete # delete extraneous files from dest dirs - #--delete-after # receiver deletes after transfer, not during - #--delete-before # receiver deletes before xfer, not during - #--delete-delay # find deletions during, delete after - #--delete-during # receiver deletes during the transfer - --delete-excluded # also delete excluded files from dest dirs - #--delete-missing-args # delete missing source args from destination - --devices # preserve device files (super-user only) - #--dirs # transfer directories without recursing - #--early-input=FILE # use FILE for daemon's early exec input - #--executability # preserve executability - #--existing # skip creating new files on receiver - #--fake-super # store/recover privileged attrs using xattrs - #--files-from=FILE # read list of source-file names from FILE - #--filter=RULE # add a file-filtering RULE - #--force # force deletion of dirs even if not empty - #--from0 # all *-from/filter files are delimited by 0s - #--fuzzy # find similar file for basis if no dest file - #--group # preserve group - #--groupmap=STRING # custom groupname mapping - #--hard-links # preserve hard links - --human-readable # output numbers in a human-readable format - #--iconv=CONVERT_SPEC # request charset conversion of filenames - #--ignore-errors # delete even if there are I/O errors - #--ignore-existing # skip updating files that exist on receiver - #--ignore-missing-args # ignore missing source args without error - #--ignore-times # don't skip files that match size and time - #--include-from=FILE # read include patterns from FILE - #--include=PATTERN # don't exclude files matching PATTERN - #--inplace # update destination files in-place - #--keep-dirlinks # treat symlinked dir on receiver as dir - #--link-dest=DIR # hardlink to files in DIR when unchanged - --links # copy symlinks as symlinks - #--list-only # list the files instead of copying them - #--log-file-format=FMT # log updates using the specified FMT - #--log-file=FILE # log what we're doing to the specified FILE - #--max-alloc=SIZE # change a limit relating to memory alloc - #--max-delete=NUM # don't delete more than NUM files - #--max-size=SIZE # don't transfer any file larger than SIZE - #--min-size=SIZE # don't transfer any file smaller than SIZE - #--mkpath # create the destination's path component - #--modify-window=NUM # set the accuracy for mod-time comparisons - #--munge-links # munge symlinks to make them safe & unusable - #--no-OPTION # turn off an implied OPTION (e.g. --no-D) - #--no-implied-dirs # don't send implied dirs with --relative - #--no-motd # suppress daemon-mode MOTD - #--numeric-ids # don't map uid/gid values by user/group name - #--omit-dir-times # omit directories from --times - #--omit-link-times # omit symlinks from --times - #--one-file-system # don't cross filesystem boundaries - #--only-write-batch=FILE # like --write-batch but w/o updating dest - #--open-noatime # avoid changing the atime on opened files - #--out-format=FORMAT # output updates using the specified FORMAT - #--outbuf=N|L|B # set out buffering to None, Line, or Block - #--owner # preserve owner (super-user only) - #--partial # keep partially transferred files - #--partial-dir=DIR # put a partially transferred file into DIR - #--password-file=FILE # read daemon-access password from FILE - --perms # preserve permissions - #--port=PORT # specify double-colon alternate port number - #--preallocate # allocate dest files before writing them - #--progress # show progress during transfer - #--protect-args # no space-splitting; wildcard chars only - #--protocol=NUM # force an older protocol version to be used - #--prune-empty-dirs # prune empty directory chains from file-list - #--quiet # suppress non-error messages - #--read-batch=FILE # read a batched update from FILE - --recursive # recurse into directories - #--relative # use relative path names - #--remote-option=OPT # send OPTION to the remote side only - #--remove-source-files # sender removes synchronized files (non-dir) - #--rsync-path=PROGRAM # specify the rsync to run on remote machine - #--safe-links # ignore symlinks that point outside the tree - #--size-only # skip files that match in size - #--skip-compress=LIST # skip compressing files with suffix in LIST - #--sockopts=OPTIONS # specify custom TCP options - #--sparse # turn sequences of nulls into sparse blocks - --specials # preserve special files - --stats # give some file-transfer stats - #--stderr=e|a|c # change stderr output mode (default: errors) - #--stop-after=MINS # Stop rsync after MINS minutes have elapsed - #--stop-at=y-m-dTh:m # Stop rsync at the specified point in time - #--suffix=SUFFIX # backup suffix (default ~ w/o --backup-dir) - #--super # receiver attempts super-user activities - #--temp-dir=DIR # create temporary files in directory DIR - #--timeout=SECONDS # set I/O timeout in seconds - --times # preserve modification times - #--update # skip files that are newer on the receiver - #--usermap=STRING # custom username mapping - #--verbose # increase verbosity - #--whole-file # copy files whole (w/o delta-xfer algorithm) - #--write-batch=FILE # write a batched update to FILE - #--write-devices # write to devices as files (implies --inplace) - #--xattrs # preserve extended attributes -) +## +## FUNCTIONS +## function check_prerequisites() { local required_commands=(awk grep head rsync screen) @@ -406,30 +281,168 @@ function load_optional_config() { export RSYNC_INFO } -return 0 # FIXME: Remove - -# -# Parse ssh connection parameters -# -RSYNC_OPTIONS+=( - --rsh - "ssh -p ${SSH_PORT} -i $(printf '%q' "${SSH_KEY}")" -) - +function collect_rsync_options() { + local default_info='progress2' + + RSYNC_OPTIONS=( + #--8-bit-output # leave high-bit chars unescaped in output + #--acls # preserve ACLs (implies --perms) + #--address=ADDRESS # bind address for outgoing socket to daemon + #--append # append data onto shorter files + #--append-verify # --append w/old data in file checksum + #--archive # archive mode; equals: '--devices --group --links --owner --perms --recursive --specials --times' + #--atimes # preserve access (use) times + #--backup # make backups (see --suffix & --backup-dir) + #--backup-dir=DIR # make backups into hierarchy based in DIR + #--block-size=SIZE # force a fixed checksum block-size + #--blocking-io # use blocking I/O for the remote shell + #--bwlimit=RATE # limit socket I/O bandwidth + #--checksum # skip based on checksum, not mod-time & size + #--checksum-choice=STR # choose the checksum algorithm + #--checksum-seed=NUM # set block/file checksum seed (advanced) + #--chmod=CHMOD # affect file and/or directory permissions + #--chown=USER:GROUP # simple username/groupname mapping + #--compare-dest=DIR # also compare destination files relative to DIR + --compress # compress file data during the transfer + #--compress-choice=STR # choose the compression algorithm + #--compress-level=NUM # explicitly set compression level + #--contimeout=SECONDS # set daemon connection timeout in seconds + #--copy-as=USER[:GROUP] # specify user & optional group for the copy + #--copy-dest=DIR # ... and include copies of unchanged files + #--copy-dirlinks # transform symlink to dir into referent dir + #--copy-links # transform symlink into referent file/dir + #--copy-unsafe-links # only "unsafe" symlinks are transformed + #--crtimes # preserve create times (newness) + #--cvs-exclude # auto-ignore files in the same way CVS does + #--debug=FLAGS # fine-grained debug verbosity + #--delay-updates # put all updated files into place at end + --delete # delete extraneous files from dest dirs + #--delete-after # receiver deletes after transfer, not during + #--delete-before # receiver deletes before xfer, not during + #--delete-delay # find deletions during, delete after + #--delete-during # receiver deletes during the transfer + --delete-excluded # also delete excluded files from dest dirs + #--delete-missing-args # delete missing source args from destination + --devices # preserve device files (super-user only) + #--dirs # transfer directories without recursing + #--early-input=FILE # use FILE for daemon's early exec input + #--executability # preserve executability + #--existing # skip creating new files on receiver + #--fake-super # store/recover privileged attrs using xattrs + #--files-from=FILE # read list of source-file names from FILE + #--filter=RULE # add a file-filtering RULE + #--force # force deletion of dirs even if not empty + #--from0 # all *-from/filter files are delimited by 0s + #--fuzzy # find similar file for basis if no dest file + #--group # preserve group + #--groupmap=STRING # custom groupname mapping + #--hard-links # preserve hard links + --human-readable # output numbers in a human-readable format + #--iconv=CONVERT_SPEC # request charset conversion of filenames + #--ignore-errors # delete even if there are I/O errors + #--ignore-existing # skip updating files that exist on receiver + #--ignore-missing-args # ignore missing source args without error + #--ignore-times # don't skip files that match size and time + #--include-from=FILE # read include patterns from FILE + #--include=PATTERN # don't exclude files matching PATTERN + #--inplace # update destination files in-place + #--keep-dirlinks # treat symlinked dir on receiver as dir + #--link-dest=DIR # hardlink to files in DIR when unchanged + --links # copy symlinks as symlinks + #--list-only # list the files instead of copying them + #--log-file-format=FMT # log updates using the specified FMT + #--log-file=FILE # log what we're doing to the specified FILE + #--max-alloc=SIZE # change a limit relating to memory alloc + #--max-delete=NUM # don't delete more than NUM files + #--max-size=SIZE # don't transfer any file larger than SIZE + #--min-size=SIZE # don't transfer any file smaller than SIZE + #--mkpath # create the destination's path component + #--modify-window=NUM # set the accuracy for mod-time comparisons + #--munge-links # munge symlinks to make them safe & unusable + #--no-OPTION # turn off an implied OPTION (e.g. --no-D) + #--no-implied-dirs # don't send implied dirs with --relative + #--no-motd # suppress daemon-mode MOTD + #--numeric-ids # don't map uid/gid values by user/group name + #--omit-dir-times # omit directories from --times + #--omit-link-times # omit symlinks from --times + #--one-file-system # don't cross filesystem boundaries + #--only-write-batch=FILE # like --write-batch but w/o updating dest + #--open-noatime # avoid changing the atime on opened files + #--out-format=FORMAT # output updates using the specified FORMAT + #--outbuf=N|L|B # set out buffering to None, Line, or Block + #--owner # preserve owner (super-user only) + #--partial # keep partially transferred files + #--partial-dir=DIR # put a partially transferred file into DIR + #--password-file=FILE # read daemon-access password from FILE + --perms # preserve permissions + #--port=PORT # specify double-colon alternate port number + #--preallocate # allocate dest files before writing them + #--progress # show progress during transfer + #--protect-args # no space-splitting; wildcard chars only + #--protocol=NUM # force an older protocol version to be used + #--prune-empty-dirs # prune empty directory chains from file-list + #--quiet # suppress non-error messages + #--read-batch=FILE # read a batched update from FILE + --recursive # recurse into directories + #--relative # use relative path names + #--remote-option=OPT # send OPTION to the remote side only + #--remove-source-files # sender removes synchronized files (non-dir) + #--rsync-path=PROGRAM # specify the rsync to run on remote machine + #--safe-links # ignore symlinks that point outside the tree + #--size-only # skip files that match in size + #--skip-compress=LIST # skip compressing files with suffix in LIST + #--sockopts=OPTIONS # specify custom TCP options + #--sparse # turn sequences of nulls into sparse blocks + --specials # preserve special files + --stats # give some file-transfer stats + #--stderr=e|a|c # change stderr output mode (default: errors) + #--stop-after=MINS # Stop rsync after MINS minutes have elapsed + #--stop-at=y-m-dTh:m # Stop rsync at the specified point in time + #--suffix=SUFFIX # backup suffix (default ~ w/o --backup-dir) + #--super # receiver attempts super-user activities + #--temp-dir=DIR # create temporary files in directory DIR + #--timeout=SECONDS # set I/O timeout in seconds + --times # preserve modification times + #--update # skip files that are newer on the receiver + #--usermap=STRING # custom username mapping + #--verbose # increase verbosity + #--whole-file # copy files whole (w/o delta-xfer algorithm) + #--write-batch=FILE # write a batched update to FILE + #--write-devices # write to devices as files (implies --inplace) + #--xattrs # preserve extended attributes + ) -# -# Dry run parameters -# -# shellcheck disable=SC2248,SC2250 -if [ $DRY_RUN_FLAG -eq 1 ] -then + # SSH connection parameters RSYNC_OPTIONS+=( - --itemize-changes # output a change-summary for all updates - --dry-run # perform a trial run with no changes made + --rsh + "ssh -p ${SSH_PORT} -i $(printf '%q' "${SSH_KEY}")" ) -else - RSYNC_OPTIONS+=() -fi + + # Info flags + if [[ "${RSYNC_INFO}" == "${__NO_VALUE__}" ]]; then + RSYNC_OPTIONS+=( + --info + "${default_info}" + ) + else + RSYNC_OPTIONS+=( + --info + "${RSYNC_INFO}" + ) + fi + + # Dry run + if [[ ${DRY_RUN_FLAG} -eq 1 ]]; then + RSYNC_OPTIONS+=( + --itemize-changes # output a change-summary for all updates + --dry-run # perform a trial run with no changes made + ) + fi + + export RSYNC_OPTIONS +} + +return 0 # FIXME: Remove # diff --git a/test/test_collect_rsync_options.bats b/test/test_collect_rsync_options.bats new file mode 100644 index 0000000..e81fe8f --- /dev/null +++ b/test/test_collect_rsync_options.bats @@ -0,0 +1,93 @@ +#!/usr/bin/env bats + +# shellcheck disable=SC2034 +setup() { + load 'test_helper/common-setup' + _common_setup + + DRY_RUN_FLAG=0 + RSYNC_INFO="${__NO_VALUE__:?}" + SSH_KEY='./path/to/key' + SSH_PORT='42' +} + +teardown() { + unset DRY_RUN_FLAG + unset RSYNC_INFO + unset SSH_KEY + unset SSH_PORT +} + +@test "collect_rsync_options : without optional config, no dry-run" { + assert_equal "${RSYNC_OPTIONS:-}" '' + + # WHEN + collect_rsync_options + + # THEN + assert_equal $? 0 + output="${RSYNC_OPTIONS[*]}" + assert_output --partial '--human-readable' + assert_output --partial '--rsh ssh -p 42 -i ./path/to/key' + assert_output --partial '--info progress2' + refute_output --partial '--itemize-changes' + refute_output --partial '--dry-run' +} + +@test "collect_rsync_options : with optional config, no dry-run" { + assert_equal "${RSYNC_OPTIONS:-}" '' + + # GIVEN + RSYNC_INFO='foo,bar' + + # WHEN + collect_rsync_options + + # THEN + assert_equal $? 0 + output="${RSYNC_OPTIONS[*]}" + assert_output --partial '--human-readable' + assert_output --partial '--rsh ssh -p 42 -i ./path/to/key' + assert_output --partial '--info foo,bar' + refute_output --partial '--itemize-changes' + refute_output --partial '--dry-run' +} + +@test "collect_rsync_options : without optional config, with dry-run" { + assert_equal "${RSYNC_OPTIONS:-}" '' + + # GIVEN + DRY_RUN_FLAG=1 + + # WHEN + collect_rsync_options + + # THEN + assert_equal $? 0 + output="${RSYNC_OPTIONS[*]}" + assert_output --partial '--human-readable' + assert_output --partial '--rsh ssh -p 42 -i ./path/to/key' + assert_output --partial '--info progress2' + assert_output --partial '--itemize-changes' + assert_output --partial '--dry-run' +} + +@test "collect_rsync_options : with optional config, with dry-run" { + assert_equal "${RSYNC_OPTIONS:-}" '' + + # GIVEN + DRY_RUN_FLAG=1 + RSYNC_INFO='lorem,ipsum' + + # WHEN + collect_rsync_options + + # THEN + assert_equal $? 0 + output="${RSYNC_OPTIONS[*]}" + assert_output --partial '--human-readable' + assert_output --partial '--rsh ssh -p 42 -i ./path/to/key' + assert_output --partial '--info lorem,ipsum' + assert_output --partial '--itemize-changes' + assert_output --partial '--dry-run' +} From e1f2187aa26fe70b7cfc86912af43c0e1f342eae Mon Sep 17 00:00:00 2001 From: Daniel Kuruc Date: Mon, 18 Apr 2022 12:33:56 +0200 Subject: [PATCH 18/18] Add `print_nice_header` function --- src/rsync_offsite_backup.sh | 30 +++++-------- test/test_print_.bats | 84 +++++++++++++++++++++++++++++++++++++ test/test_print_usage.bats | 35 ---------------- 3 files changed, 95 insertions(+), 54 deletions(-) create mode 100644 test/test_print_.bats delete mode 100644 test/test_print_usage.bats diff --git a/src/rsync_offsite_backup.sh b/src/rsync_offsite_backup.sh index 6d1191c..97ae1f7 100755 --- a/src/rsync_offsite_backup.sh +++ b/src/rsync_offsite_backup.sh @@ -442,8 +442,18 @@ function collect_rsync_options() { export RSYNC_OPTIONS } -return 0 # FIXME: Remove +function print_nice_header() { + local _dry_run="$([[ ${DRY_RUN_FLAG} -eq 1 ]] && echo 'yes' || echo 'no')" + local _remote_target="${SSH_USER}@${SSH_HOST}:${PATH_REMOTE}" + + _header "STARTING RSYNC SESSION" \ + "Job name: ${JOB_NAME}" \ + "Dry run: ${_dry_run}" \ + "Local source: ${PATH_SOURCE}" \ + "Remote target: ${_remote_target}" +} +return 0 # FIXME: Remove # # Finally, tell rsync about paths to transfer @@ -453,24 +463,6 @@ RSYNC_OPTIONS+=( "${SSH_USER}@${SSH_HOST}:$(printf '%q' "${PATH_REMOTE}")" ) - -# -# Print nice header -# -_printf_line=' ' -# shellcheck disable=SC2250 -_dry_run="$([[ $DRY_RUN_FLAG -eq 1 ]] && echo 'yes' || echo 'no')" -_remote_target="${SSH_USER}@${SSH_HOST}:${PATH_REMOTE}" -echo "┌────────────────────────────────────────────────────────────────────────────┐" -echo "│ STARTING RSYNC SESSION │" -echo "├────────────────────────────────────────────────────────────────────────────┤" -printf "│ Job name: %s%s│\n" "${JOB_NAME}" "${_printf_line:${#JOB_NAME}}" -printf "│ Dry run: %s%s│\n" "${_dry_run}" "${_printf_line:${#_dry_run}}" -printf "│ Local source: %s%s│\n" "${PATH_SOURCE}" "${_printf_line:${#PATH_SOURCE}}" -printf "│ Remote target: %s%s│\n" "${_remote_target}" "${_printf_line:${#_remote_target}}" -echo "└────────────────────────────────────────────────────────────────────────────┘" - - # # Dry run message # diff --git a/test/test_print_.bats b/test/test_print_.bats new file mode 100644 index 0000000..0db6796 --- /dev/null +++ b/test/test_print_.bats @@ -0,0 +1,84 @@ +#!/usr/bin/env bats + +setup() { + load 'test_helper/common-setup' + _common_setup + + # MOCK / GIVEN + export DASEL_VER='42' + export ENVSUBST_VER='42' + export RSYNC_PROTOCOL_VER='42' + export RSYNC_VER='42' + export YQ_VER='42' + + export SSH_USER='foo' + export SSH_HOST='bar' + export PATH_REMOTE='lorem/' + export JOB_NAME='ipsum' + export PATH_SOURCE='/given/path/' +} + +teardown() { + unset DASEL_VER + unset ENVSUBST_VER + unset RSYNC_PROTOCOL_VER + unset RSYNC_VER + unset YQ_VER + + unset DRY_RUN_FLAG + unset JOB_NAME + unset PATH_REMOTE + unset PATH_SOURCE + unset SSH_HOST + unset SSH_USER +} + +# print_usage ------------------------------------------------------------------ + +@test "print_usage result" { + # WHEN + run print_usage + + # THEN + assert_success + assert_output --partial 'Usage:' + assert_output --partial 'Options:' + assert_output --partial 'dasel:' + assert_output --partial 'envsubst:' + assert_output --partial 'rsync:' + assert_output --partial 'yq:' +} + +# print_nice_header ------------------------------------------------------------ + +@test "print_nice_header : when dry run is disabled" { + # GIVEN + export DRY_RUN_FLAG=0 + + # WHEN + run print_nice_header + + # THEN + assert_success + assert_output --partial 'STARTING RSYNC SESSION' + assert_output --partial 'Job name: ipsum' + assert_output --partial 'Dry run: no' + assert_output --partial 'Local source: /given/path/' + assert_output --partial 'Remote target: foo@bar:lorem/' +} + +@test "print_nice_header : when dry run is enabled" { + # GIVEN + export DRY_RUN_FLAG=1 + + # WHEN + run print_nice_header + + # THEN + assert_success + assert_output --partial 'STARTING RSYNC SESSION' + assert_output --partial 'Job name: ipsum' + assert_output --partial 'Dry run: yes' + assert_output --partial 'Local source: /given/path/' + assert_output --partial 'Remote target: foo@bar:lorem/' +} diff --git a/test/test_print_usage.bats b/test/test_print_usage.bats deleted file mode 100644 index 7214b89..0000000 --- a/test/test_print_usage.bats +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bats - -setup() { - load 'test_helper/common-setup' - _common_setup - - # MOCK - export DASEL_VER='42' - export ENVSUBST_VER='42' - export RSYNC_PROTOCOL_VER='42' - export RSYNC_VER='42' - export YQ_VER='42' -} - -teardown() { - unset DASEL_VER - unset ENVSUBST_VER - unset RSYNC_PROTOCOL_VER - unset RSYNC_VER - unset YQ_VER -} - -@test "print_usage result" { - # WHEN - run print_usage - - # THEN - assert_success - assert_output --partial 'Usage:' - assert_output --partial 'Options:' - assert_output --partial 'dasel:' - assert_output --partial 'envsubst:' - assert_output --partial 'rsync:' - assert_output --partial 'yq:' -}