-
Notifications
You must be signed in to change notification settings - Fork 184
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add a marker .download file to validate the contents in cache directories. Previously only the existence of the directory was used, so if the download was aborted the cache directory had to be deleted manually if this occurred (with a likely cryptic error message). If the .download check file does not exist, the directory will be deleted and downloaded again. It is also possible to check the contents with a checksum. If not matching, the directory will be deleted and downloaded again. For Git repos the repos can be deleted if the status is not clean, a checksum is not relevant (but used in the tests).
- Loading branch information
Showing
4 changed files
with
313 additions
and
6 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,63 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Script to checksum contents recursively in a directory | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
|
|
||
| function usage { | ||
| echo | ||
| echo "Checksum the contents of a directory" | ||
| echo "Usage: $0 [-d <directory>]" | ||
| echo "" | ||
| echo " -d directory Default '.'" | ||
| echo " -h Help, this message" | ||
| echo " -t Use alternative tar method (requires zstd binary)" | ||
| echo " -v Verbose output" | ||
| } | ||
|
|
||
| dir=. | ||
| use_tar= | ||
| # sha512 is faster than sha256 for large files, sha1 is even faster | ||
| SHA_ALGORITHM=sha512sum | ||
| if [[ "$OSTYPE" != "darwin"* ]]; then | ||
| # Some overrides required for macos | ||
| # Note also that 'xargs --max-procs' must be written as 'xargs -P' | ||
| SHA_ALGORITHM="shasum -a 512" | ||
| alias nproc="sysctl -n hw.logicalcpu" | ||
| fi | ||
|
|
||
| while getopts "d:htv" o; do | ||
| case "${o}" in | ||
| d) | ||
| dir=${OPTARG} | ||
| ;; | ||
| h) | ||
| usage | ||
| exit 0 | ||
| ;; | ||
| t) | ||
| use_tar=1 | ||
| ;; | ||
| v) | ||
| set -x | ||
| ;; | ||
| *) | ||
| echo "Incorrect argument switch" | ||
| usage | ||
| exit 1 | ||
| ;; | ||
| esac | ||
| done | ||
| shift "$((OPTIND-1))" | ||
|
|
||
| cd $dir | ||
| if [ ! -z $use_tar ]; then | ||
| # This is faster for single threads but requires more memory and requires the separate zstd binary | ||
| # For a 3 GB data this is 3s vs 'find' below: 5s (one thread) below, 2.5s with 28 threads, 0.7s with 100 files on each line | ||
| # Without --fast, just ZSTD_CLEVEL=1 ZSTD_NBTHREADS=0 is about 6s | ||
| tar -I "zstd --fast -1 -T0" -cf - . | $SHA_ALGORITHM | cut -f1 -d ' ' | ||
| else | ||
| # In general, there is no point in checksumming Git repos, filter .git here as this is used in tests | ||
| find . \( -name .git -prune \) -o -type f -print0 | xargs -n 100 -P=$(nproc) -0 $SHA_ALGORITHM | sort -k 2 | $SHA_ALGORITHM | cut -f1 -d ' ' | ||
| fi |