diff --git a/.github/workflows/generic-dev.yml b/.github/workflows/generic-dev.yml index 87b2507284a..ae4ee7e9f53 100644 --- a/.github/workflows/generic-dev.yml +++ b/.github/workflows/generic-dev.yml @@ -30,6 +30,9 @@ jobs: test: runs-on: ubuntu-latest + env: + DEVNULLRIGHTS: 1 + READFROMBLOCKDEVICE: 1 steps: - uses: actions/checkout@v2 - name: make test diff --git a/programs/fileio.c b/programs/fileio.c index 604084cc707..eb06197f192 100644 --- a/programs/fileio.c +++ b/programs/fileio.c @@ -324,6 +324,7 @@ struct FIO_prefs_s { int excludeCompressedFiles; int patchFromMode; int contentSize; + int allowBlockDevices; }; /*-************************************* @@ -384,6 +385,7 @@ FIO_prefs_t* FIO_createPreferences(void) ret->testMode = 0; ret->literalCompressionMode = ZSTD_lcm_auto; ret->excludeCompressedFiles = 0; + ret->allowBlockDevices = 0; return ret; } @@ -451,6 +453,8 @@ void FIO_setNbWorkers(FIO_prefs_t* const prefs, int nbWorkers) { void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles) { prefs->excludeCompressedFiles = excludeCompressedFiles; } +void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices) { prefs->allowBlockDevices = allowBlockDevices; } + void FIO_setBlockSize(FIO_prefs_t* const prefs, int blockSize) { if (blockSize && prefs->nbWorkers==0) DISPLAYLEVEL(2, "Setting block size is useless in single-thread mode \n"); @@ -593,11 +597,12 @@ static int FIO_removeFile(const char* path) } /** FIO_openSrcFile() : - * condition : `srcFileName` must be non-NULL. + * condition : `srcFileName` must be non-NULL. `prefs` may be NULL. * @result : FILE* to `srcFileName`, or NULL if it fails */ -static FILE* FIO_openSrcFile(const char* srcFileName) +static FILE* FIO_openSrcFile(const FIO_prefs_t* const prefs, const char* srcFileName) { stat_t statbuf; + int allowBlockDevices = prefs != NULL ? prefs->allowBlockDevices : 0; assert(srcFileName != NULL); if (!strcmp (srcFileName, stdinmark)) { DISPLAYLEVEL(4,"Using stdin for input \n"); @@ -613,6 +618,7 @@ static FILE* FIO_openSrcFile(const char* srcFileName) if (!UTIL_isRegularFileStat(&statbuf) && !UTIL_isFIFOStat(&statbuf) + && !(allowBlockDevices && UTIL_isBlockDevStat(&statbuf)) ) { DISPLAYLEVEL(1, "zstd: %s is not a regular file -- ignored \n", srcFileName); @@ -1708,7 +1714,7 @@ FIO_compressFilename_srcFile(FIO_ctx_t* const fCtx, return 0; } - ress.srcFile = FIO_openSrcFile(srcFileName); + ress.srcFile = FIO_openSrcFile(prefs, srcFileName); if (ress.srcFile == NULL) return 1; /* srcFile could not be opened */ result = FIO_compressFilename_dstFile(fCtx, prefs, ress, dstFileName, srcFileName, compressionLevel); @@ -2571,7 +2577,7 @@ static int FIO_decompressSrcFile(FIO_ctx_t* const fCtx, FIO_prefs_t* const prefs return 1; } - srcFile = FIO_openSrcFile(srcFileName); + srcFile = FIO_openSrcFile(prefs, srcFileName); if (srcFile==NULL) return 1; ress.srcBufferLoaded = 0; @@ -2921,7 +2927,7 @@ static InfoError getFileInfo_fileConfirmed(fileInfo_t* info, const char* inFileName) { InfoError status; - FILE* const srcFile = FIO_openSrcFile(inFileName); + FILE* const srcFile = FIO_openSrcFile(NULL, inFileName); ERROR_IF(srcFile == NULL, info_file_error, "Error: could not open source file %s", inFileName); info->compressedSize = UTIL_getFileSize(inFileName); diff --git a/programs/fileio.h b/programs/fileio.h index 1f5db4b5297..2a5591c3b18 100644 --- a/programs/fileio.h +++ b/programs/fileio.h @@ -103,6 +103,7 @@ void FIO_setLiteralCompressionMode( void FIO_setNoProgress(unsigned noProgress); void FIO_setNotificationLevel(int level); void FIO_setExcludeCompressedFile(FIO_prefs_t* const prefs, int excludeCompressedFiles); +void FIO_setAllowBlockDevices(FIO_prefs_t* const prefs, int allowBlockDevices); void FIO_setPatchFromMode(FIO_prefs_t* const prefs, int value); void FIO_setContentSize(FIO_prefs_t* const prefs, int value); diff --git a/programs/util.c b/programs/util.c index 0f857019af7..0d40a394e38 100644 --- a/programs/util.c +++ b/programs/util.c @@ -269,6 +269,17 @@ int UTIL_isFIFOStat(const stat_t* statbuf) return 0; } +/* UTIL_isBlockDevStat : distinguish named pipes */ +int UTIL_isBlockDevStat(const stat_t* statbuf) +{ +/* macro guards, as defined in : https://linux.die.net/man/2/lstat */ +#if PLATFORM_POSIX_VERSION >= 200112L + if (S_ISBLK(statbuf->st_mode)) return 1; +#endif + (void)statbuf; + return 0; +} + int UTIL_isLink(const char* infilename) { /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ diff --git a/programs/util.h b/programs/util.h index af3778a2f2e..86743ce57cb 100644 --- a/programs/util.h +++ b/programs/util.h @@ -143,6 +143,7 @@ int UTIL_setFileStat(const char* filename, const stat_t* statbuf); int UTIL_isRegularFileStat(const stat_t* statbuf); int UTIL_isDirectoryStat(const stat_t* statbuf); int UTIL_isFIFOStat(const stat_t* statbuf); +int UTIL_isBlockDevStat(const stat_t* statbuf); U64 UTIL_getFileSizeStat(const stat_t* statbuf); /** diff --git a/programs/zstd.1 b/programs/zstd.1 index 1b7c88d8a95..c73ea636dc4 100644 --- a/programs/zstd.1 +++ b/programs/zstd.1 @@ -1,5 +1,7 @@ +.\" generated with Ronn/v0.7.3 +.\" http://github.com/rtomayko/ronn/tree/0.7.3 . -.TH "ZSTD" "1" "March 2021" "zstd 1.4.9" "User Commands" +.TH "ZSTD" "1" "May 2021" "zstd 1.4.10" "User Commands" . .SH "NAME" \fBzstd\fR \- zstd, zstdmt, unzstd, zstdcat \- Compress or decompress \.zst files @@ -156,7 +158,7 @@ This is also used during compression when using with \-\-patch\-from=\. In this \fB\-o FILE\fR: save result into \fBFILE\fR . .IP "\(bu" 4 -\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, etc\. +\fB\-f\fR, \fB\-\-force\fR: disable input and output checks\. Allows overwriting existing files, input from console, output to stdout, operating on links, block devices, etc\. . .IP "\(bu" 4 \fB\-c\fR, \fB\-\-stdout\fR: force write to standard output, even if it is the console diff --git a/programs/zstd.1.md b/programs/zstd.1.md index 0b259b668c6..333b594f140 100644 --- a/programs/zstd.1.md +++ b/programs/zstd.1.md @@ -202,7 +202,7 @@ the last one takes effect. save result into `FILE` * `-f`, `--force`: disable input and output checks. Allows overwriting existing files, input - from console, output to stdout, operating on links, etc. + from console, output to stdout, operating on links, block devices, etc. * `-c`, `--stdout`: force write to standard output, even if it is the console * `--[no-]sparse`: diff --git a/programs/zstdcli.c b/programs/zstdcli.c index d9d2c701e48..b0ca7afc022 100644 --- a/programs/zstdcli.c +++ b/programs/zstdcli.c @@ -148,7 +148,8 @@ static void usage(FILE* f, const char* programName) DISPLAY_F(f, " -D DICT: use DICT as Dictionary for compression or decompression \n"); DISPLAY_F(f, " -o file: result stored into `file` (only 1 output file) \n"); DISPLAY_F(f, " -f : disable input and output checks. Allows overwriting existing files,\n"); - DISPLAY_F(f, " input from console, output to stdout, operating on links, etc.\n"); + DISPLAY_F(f, " input from console, output to stdout, operating on links,\n"); + DISPLAY_F(f, " block devices, etc.\n"); DISPLAY_F(f, "--rm : remove source file(s) after successful de/compression \n"); DISPLAY_F(f, " -k : preserve source file(s) (default) \n"); DISPLAY_F(f, " -h/-H : display help/long help and exit \n"); @@ -724,6 +725,7 @@ int main(int const argCount, const char* argv[]) { int argNb, followLinks = 0, + allowBlockDevices = 0, forceStdin = 0, forceStdout = 0, hasStdout = 0, @@ -838,7 +840,7 @@ int main(int const argCount, const char* argv[]) if (!strcmp(argument, "--compress")) { operation=zom_compress; continue; } if (!strcmp(argument, "--decompress")) { operation=zom_decompress; continue; } if (!strcmp(argument, "--uncompress")) { operation=zom_decompress; continue; } - if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; continue; } + if (!strcmp(argument, "--force")) { FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; continue; } if (!strcmp(argument, "--version")) { printVersion(); CLEAN_RETURN(0); } if (!strcmp(argument, "--help")) { usage_advanced(programName); CLEAN_RETURN(0); } if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } @@ -1024,7 +1026,7 @@ int main(int const argCount, const char* argv[]) case 'D': argument++; NEXT_FIELD(dictFileName); break; /* Overwrite */ - case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; argument++; break; + case 'f': FIO_overwriteMode(prefs); forceStdin=1; forceStdout=1; followLinks=1; allowBlockDevices=1; argument++; break; /* Verbose mode */ case 'v': g_displayLevel++; argument++; break; @@ -1331,6 +1333,7 @@ int main(int const argCount, const char* argv[]) FIO_setNbFilesTotal(fCtx, (int)filenames->tableSize); FIO_determineHasStdinInput(fCtx, filenames); FIO_setNotificationLevel(g_displayLevel); + FIO_setAllowBlockDevices(prefs, allowBlockDevices); FIO_setPatchFromMode(prefs, patchFromDictFileName != NULL); if (memLimit == 0) { if (compressionParams.windowLog == 0) { diff --git a/tests/playTests.sh b/tests/playTests.sh index 04a90e3c2ee..f002d34ecae 100755 --- a/tests/playTests.sh +++ b/tests/playTests.sh @@ -459,6 +459,22 @@ if [ -n "$DEVNULLRIGHTS" ] ; then ls -las $INTOVOID | grep "rw-rw-rw-" fi +if [ -n "$READFROMBLOCKDEVICE" ] ; then + # This creates a temporary block device, which is only possible on unix-y + # systems, is somewhat invasive, and requires sudo. For these reasons, you + # have to specifically ask for this test. + println "\n===> checking that zstd can read from a block device" + datagen -g65536 > tmp.img + sudo losetup -fP tmp.img + LOOP_DEV=$(losetup -a | grep 'tmp\.img' | cut -f1 -d:) + [ -z "$LOOP_DEV" ] && die "failed to get loopback device" + sudoZstd $LOOP_DEV -c > tmp.img.zst && die "should fail without -f" + sudoZstd -f $LOOP_DEV -c > tmp.img.zst + zstd -d tmp.img.zst -o tmp.img.copy + sudo losetup -d $LOOP_DEV + $DIFF -s tmp.img tmp.img.copy || die "round trip failed" + rm -f tmp.img tmp.img.zst tmp.img.copy +fi println "\n===> compress multiple files into an output directory, --output-dir-flat" println henlo > tmp1