diff --git a/CMakeLists.txt b/CMakeLists.txt index 3af9e8176..f4184616f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ option(ENABLE_WINDOWS_CRYPTO "Enable use of Windows cryptography libraries" ON) option(ENABLE_BZIP2 "Enable use of BZip2" ON) option(ENABLE_LZMA "Enable use of LZMA" ON) +option(ENABLE_ZSTD "Enable use of Zstandard" ON) option(BUILD_TOOLS "Build tools in the src directory (zipcmp, zipmerge, ziptool)" ON) option(BUILD_REGRESS "Build regression tests" ON) @@ -203,6 +204,14 @@ if(ENABLE_LZMA) endif(LIBLZMA_FOUND) endif(ENABLE_LZMA) +if(ENABLE_ZSTD) + find_package(LibZSTD) + if(LIBZSTD_FOUND) + set(HAVE_LIBZSTD 1) + else() + message(WARNING "-- zstd library not found; zstd support disabled") + endif(LIBZSTD_FOUND) +endif(ENABLE_ZSTD) if (COMMONCRYPTO_FOUND) set(HAVE_CRYPTO 1) diff --git a/THANKS b/THANKS index 7a16c62c3..40027b65a 100644 --- a/THANKS +++ b/THANKS @@ -32,6 +32,7 @@ Elvis Angelaccio Erwin Haid Eun-cheol Joo Florian Delizy +Force Charlie François Simon Frederik Ramm gk7huki diff --git a/cmake-config.h.in b/cmake-config.h.in index 2df6f8296..544adbde9 100644 --- a/cmake-config.h.in +++ b/cmake-config.h.in @@ -29,6 +29,7 @@ #cmakedefine HAVE_GNUTLS #cmakedefine HAVE_LIBBZ2 #cmakedefine HAVE_LIBLZMA +#cmakedefine HAVE_LIBZSTD #cmakedefine HAVE_LOCALTIME_R #cmakedefine HAVE_MBEDTLS #cmakedefine HAVE_MKSTEMP diff --git a/cmake/FindLibZSTD.cmake b/cmake/FindLibZSTD.cmake new file mode 100644 index 000000000..8ae328afd --- /dev/null +++ b/cmake/FindLibZSTD.cmake @@ -0,0 +1,39 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# - Try to find Facebook zstd library +# This will define +# ZSTD_FOUND +# ZSTD_INCLUDE_DIR +# ZSTD_LIBRARY +# + +find_path(ZSTD_INCLUDE_DIR NAMES zstd.h) + +find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd) +find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static) + +include(SelectLibraryConfigurations) +SELECT_LIBRARY_CONFIGURATIONS(ZSTD) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(LibZSTD + FOUND_VAR LIBZSTD_FOUND + REQUIRED_VARS + ZSTD_LIBRARY ZSTD_INCLUDE_DIR + VERSION_VAR ZSTD_VERSION +) + +mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 082203d04..d631e42a6 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -146,6 +146,11 @@ if(HAVE_LIBLZMA) target_link_libraries(zip PRIVATE LibLZMA::LibLZMA) endif() +if(HAVE_LIBZSTD) + target_sources(zip PRIVATE zip_algorithm_zstd.c) + target_link_libraries(zip PRIVATE zstd) +endif() + if(HAVE_COMMONCRYPTO) target_sources(zip PRIVATE zip_crypto_commoncrypto.c) elseif(HAVE_WINDOWS_CRYPTO) diff --git a/lib/zip.h b/lib/zip.h index a90776df2..092789404 100644 --- a/lib/zip.h +++ b/lib/zip.h @@ -135,6 +135,7 @@ extern "C" { #define ZIP_ER_TELL 30 /* S Tell error */ #define ZIP_ER_COMPRESSED_DATA 31 /* N Compressed data invalid */ #define ZIP_ER_CANCELLED 32 /* N Operation cancelled */ +#define ZIP_ER_ZSTD 33 /*Z Zstd error */ /* type of system error value */ @@ -164,6 +165,7 @@ extern "C" { #define ZIP_CM_TERSE 18 /* compressed using IBM TERSE (new) */ #define ZIP_CM_LZ77 19 /* IBM LZ77 z Architecture (PFS) */ #define ZIP_CM_LZMA2 33 +#define ZIP_CM_ZSTD 93 /* WinZip zstd but APPNOTE.TXT update */ #define ZIP_CM_XZ 95 /* XZ compressed data */ #define ZIP_CM_JPEG 96 /* Compressed Jpeg data */ #define ZIP_CM_WAVPACK 97 /* WavPack compressed data */ diff --git a/lib/zip_algorithm_zstd.c b/lib/zip_algorithm_zstd.c new file mode 100644 index 000000000..0c7712ea2 --- /dev/null +++ b/lib/zip_algorithm_zstd.c @@ -0,0 +1,221 @@ +/* + zip_algorithm_zstd.c -- zstd (de)compression routines + Copyright (C) 2017-2019 Dieter Baron and Thomas Klausner + + This file is part of libzip, a library to manipulate ZIP archives. + The authors can be contacted at + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + 3. The names of the authors may not be used to endorse or promote + products derived from this software without specific prior + written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE + GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER + IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN + IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "zipint.h" + +#include +#include +#include + +struct ctx { + zip_error_t *error; + bool compress; + zip_uint32_t compression_flags; + bool end_of_input; + ZSTD_DStream *zdstream; + ZSTD_CStream *zcstream; + ZSTD_outBuffer out; + ZSTD_inBuffer in; +}; + + +static void * +allocate(bool compress, int compression_flags, zip_error_t *error, zip_uint16_t method) { + struct ctx *ctx; + + if (compression_flags < 0) { + zip_error_set(error, ZIP_ER_INVAL, 0); + return NULL; + } + + if ((ctx = (struct ctx *)malloc(sizeof(*ctx))) == NULL) { + zip_error_set(error, ZIP_ER_MEMORY, 0); + return NULL; + } + + ctx->error = error; + ctx->compress = compress; + ctx->compression_flags = (zip_uint32_t)compression_flags; + ctx->end_of_input = false; + + return ctx; +} + + +static void * +compress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) { + return allocate(true, compression_flags, error, method); +} + + +static void * +decompress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) { + return allocate(false, compression_flags, error, method); +} + + +static void +deallocate(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + free(ctx); +} + + +static zip_uint16_t +general_purpose_bit_flags(void *ud) { + /* struct ctx *ctx = (struct ctx *)ud; */ + return 0; +} + +static int +map_error(size_t ret) { + return 0; +} + + +static bool +start(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + ctx->in.src = NULL; + ctx->in.pos = 0; + ctx->in.size = 0; + ctx->out.dst = NULL; + ctx->out.pos = 0; + ctx->out.size = 0; + if (ctx->compress) { + ctx->zcstream = ZSTD_createCStream(); + } + else { + ctx->zdstream = ZSTD_createDStream(); + } + + return true; +} + + +static bool +end(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + if (ctx->compress) { + ZSTD_freeCStream(ctx->zcstream); + ctx->zcstream = NULL; + } + else { + ZSTD_freeDStream(ctx->zdstream); + ctx->zdstream = NULL; + } + + return true; +} + + +static bool +input(void *ud, zip_uint8_t *data, zip_uint64_t length) { + struct ctx *ctx = (struct ctx *)ud; + if (ctx->in.pos != 0 && ctx->in.pos != ctx->in.size) { + zip_error_set(ctx->error, ZIP_ER_INVAL, 0); + return false; + } + ctx->in.src = (const void *)data; + ctx->in.size = length; + ctx->in.pos = 0; + return true; +} + + +static void +end_of_input(void *ud) { + struct ctx *ctx = (struct ctx *)ud; + + ctx->end_of_input = true; +} + + +static zip_compression_status_t +process(void *ud, zip_uint8_t *data, zip_uint64_t *length) { + struct ctx *ctx = (struct ctx *)ud; + + size_t ret; + + ctx->out.dst = data; + ctx->out.pos = 0; + ctx->out.size = ZIP_MIN(UINT_MAX, *length); + + if (ctx->compress) { + ret = ZSTD_compressStream(ctx->zcstream, &ctx->out, &ctx->in); + } + else { + ret = ZSTD_decompressStream(ctx->zdstream, &ctx->out, &ctx->in); + } + if (ZSTD_isError(ret)) { + zip_error_set(ctx->error, ZIP_ER_ZSTD, (int)ret); + return ZIP_COMPRESSION_ERROR; + } + *length = ctx->out.pos; + if (ctx->out.pos == 0) { + return ZIP_COMPRESSION_NEED_DATA; + } + if (ctx->in.size == 0) { + return ZIP_COMPRESSION_END; + } + return ZIP_COMPRESSION_OK; +} + +/* clang-format off */ + +zip_compression_algorithm_t zip_algorithm_zstd_compress = { + compress_allocate, + deallocate, + general_purpose_bit_flags, + 20, + start, + end, + input, + end_of_input, + process +}; + + +zip_compression_algorithm_t zip_algorithm_zstd_decompress = { + decompress_allocate, + deallocate, + general_purpose_bit_flags, + 20, + start, + end, + input, + end_of_input, + process +}; + +/* clang-format on */ \ No newline at end of file diff --git a/lib/zip_source_compress.c b/lib/zip_source_compress.c index e49929f56..1c4c8bd7e 100644 --- a/lib/zip_source_compress.c +++ b/lib/zip_source_compress.c @@ -75,6 +75,9 @@ static struct implementation implementations[] = { */ {ZIP_CM_XZ, &zip_algorithm_xz_compress, &zip_algorithm_xz_decompress}, #endif +#if defined(HAVE_LIBZSTD) + {ZIP_CM_ZSTD, &zip_algorithm_zstd_compress, &zip_algorithm_zstd_decompress}, +#endif }; diff --git a/lib/zipint.h b/lib/zipint.h index 99c180613..139eb9839 100644 --- a/lib/zipint.h +++ b/lib/zipint.h @@ -153,6 +153,8 @@ extern zip_compression_algorithm_t zip_algorithm_deflate_compress; extern zip_compression_algorithm_t zip_algorithm_deflate_decompress; extern zip_compression_algorithm_t zip_algorithm_xz_compress; extern zip_compression_algorithm_t zip_algorithm_xz_decompress; +extern zip_compression_algorithm_t zip_algorithm_zstd_compress; +extern zip_compression_algorithm_t zip_algorithm_zstd_decompress; /* This API is not final yet, but we need it internally, so it's private for now. */ diff --git a/src/ziptool.c b/src/ziptool.c index cdcecc7d6..0c28bde63 100644 --- a/src/ziptool.c +++ b/src/ziptool.c @@ -652,6 +652,11 @@ get_compression_method(const char *arg) { else if (strcasecmp(arg, "xz") == 0) return ZIP_CM_XZ; +#endif +#if defined(HAVE_LIBZSTD) + else if (strcasecmp(arg, "zstd") == 0) + return ZIP_CM_ZSTD; + #endif else if (strcasecmp(arg, "unknown") == 0) return 100;