zip adds zstd compression algorithm support

CMakeLists.txt

@@ -17,6 +17,7 @@ option(ENABLE_WINDOWS_CRYPTO "Enable use of Windows cryptography libraries" ON)
 
 option(ENABLE_BZIP2 "Enable use of BZip2" ON)
 option(ENABLE_LZMA "Enable use of LZMA" ON)
+option(ENABLE_ZSTD "Enable use of Zstandard" ON)
 
 option(BUILD_TOOLS "Build tools in the src directory (zipcmp, zipmerge, ziptool)" ON)
 option(BUILD_REGRESS "Build regression tests" ON)
@@ -203,6 +204,14 @@ if(ENABLE_LZMA)
   endif(LIBLZMA_FOUND)
 endif(ENABLE_LZMA)
 
+if(ENABLE_ZSTD)
+  find_package(LibZSTD)
+  if(LIBZSTD_FOUND)
+    set(HAVE_LIBZSTD 1)
+  else()
+    message(WARNING "-- zstd library not found; zstd support disabled")
+  endif(LIBZSTD_FOUND)
+endif(ENABLE_ZSTD)
 
 if (COMMONCRYPTO_FOUND)
   set(HAVE_CRYPTO 1)

THANKS

@@ -32,6 +32,7 @@ Elvis Angelaccio
 Erwin Haid <erwin.haid@gmx.de>
 Eun-cheol Joo
 Florian Delizy <florian.delizy@gmail.com>
+Force Charlie <charlieio@outlook.com>
 François Simon <AT.GFI.Francois.SIMON@sesam-vitale.fr>
 Frederik Ramm <frederik@remote.org>
 gk7huki <gk7huki@gmail.com>

cmake-config.h.in

@@ -29,6 +29,7 @@
 #cmakedefine HAVE_GNUTLS
 #cmakedefine HAVE_LIBBZ2
 #cmakedefine HAVE_LIBLZMA
+#cmakedefine HAVE_LIBZSTD
 #cmakedefine HAVE_LOCALTIME_R
 #cmakedefine HAVE_MBEDTLS
 #cmakedefine HAVE_MKSTEMP

cmake/FindLibZSTD.cmake

@@ -0,0 +1,39 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#
+# - Try to find Facebook zstd library
+# This will define
+# ZSTD_FOUND
+# ZSTD_INCLUDE_DIR
+# ZSTD_LIBRARY
+#
+
+find_path(ZSTD_INCLUDE_DIR NAMES zstd.h)
+
+find_library(ZSTD_LIBRARY_DEBUG NAMES zstdd zstd_staticd)
+find_library(ZSTD_LIBRARY_RELEASE NAMES zstd zstd_static)
+
+include(SelectLibraryConfigurations)
+SELECT_LIBRARY_CONFIGURATIONS(ZSTD)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(LibZSTD
+    FOUND_VAR LIBZSTD_FOUND
+    REQUIRED_VARS 
+      ZSTD_LIBRARY ZSTD_INCLUDE_DIR
+    VERSION_VAR ZSTD_VERSION
+)
+
+mark_as_advanced(ZSTD_INCLUDE_DIR ZSTD_LIBRARY)

lib/CMakeLists.txt

@@ -146,6 +146,11 @@ if(HAVE_LIBLZMA)
   target_link_libraries(zip PRIVATE LibLZMA::LibLZMA)
 endif()
 
+if(HAVE_LIBZSTD)
+  target_sources(zip PRIVATE zip_algorithm_zstd.c)
+  target_link_libraries(zip PRIVATE zstd)
+endif()
+
 if(HAVE_COMMONCRYPTO)
   target_sources(zip PRIVATE zip_crypto_commoncrypto.c)
 elseif(HAVE_WINDOWS_CRYPTO)

lib/zip.h

@@ -135,6 +135,7 @@ extern "C" {
 #define ZIP_ER_TELL 30            /* S Tell error */
 #define ZIP_ER_COMPRESSED_DATA 31 /* N Compressed data invalid */
 #define ZIP_ER_CANCELLED 32       /* N Operation cancelled */
+#define ZIP_ER_ZSTD 33            /*Z Zstd error */
 
 /* type of system error value */
 
@@ -164,6 +165,7 @@ extern "C" {
 #define ZIP_CM_TERSE 18 /* compressed using IBM TERSE (new) */
 #define ZIP_CM_LZ77 19  /* IBM LZ77 z Architecture (PFS) */
 #define ZIP_CM_LZMA2 33
+#define ZIP_CM_ZSTD 93    /* WinZip zstd but APPNOTE.TXT update */
 #define ZIP_CM_XZ 95      /* XZ compressed data */
 #define ZIP_CM_JPEG 96    /* Compressed Jpeg data */
 #define ZIP_CM_WAVPACK 97 /* WavPack compressed data */

lib/zip_algorithm_zstd.c

@@ -0,0 +1,221 @@
+/*
+  zip_algorithm_zstd.c -- zstd (de)compression routines
+  Copyright (C) 2017-2019 Dieter Baron and Thomas Klausner
+
+  This file is part of libzip, a library to manipulate ZIP archives.
+  The authors can be contacted at <libzip@nih.at>
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions
+  are met:
+  1. Redistributions of source code must retain the above copyright
+     notice, this list of conditions and the following disclaimer.
+  2. Redistributions in binary form must reproduce the above copyright
+     notice, this list of conditions and the following disclaimer in
+     the documentation and/or other materials provided with the
+     distribution.
+  3. The names of the authors may not be used to endorse or promote
+     products derived from this software without specific prior
+     written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
+  OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+  ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY
+  DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+  GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+  IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+  OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
+  IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+#include "zipint.h"
+
+#include <zstd.h>
+#include <limits.h>
+#include <stdlib.h>
+
+struct ctx {
+    zip_error_t *error;
+    bool compress;
+    zip_uint32_t compression_flags;
+    bool end_of_input;
+    ZSTD_DStream *zdstream;
+    ZSTD_CStream *zcstream;
+    ZSTD_outBuffer out;
+    ZSTD_inBuffer in;
+};
+
+
+static void *
+allocate(bool compress, int compression_flags, zip_error_t *error, zip_uint16_t method) {
+    struct ctx *ctx;
+
+    if (compression_flags < 0) {
+	zip_error_set(error, ZIP_ER_INVAL, 0);
+	return NULL;
+    }
+
+    if ((ctx = (struct ctx *)malloc(sizeof(*ctx))) == NULL) {
+	zip_error_set(error, ZIP_ER_MEMORY, 0);
+	return NULL;
+    }
+
+    ctx->error = error;
+    ctx->compress = compress;
+    ctx->compression_flags = (zip_uint32_t)compression_flags;
+    ctx->end_of_input = false;
+
+    return ctx;
+}
+
+
+static void *
+compress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) {
+    return allocate(true, compression_flags, error, method);
+}
+
+
+static void *
+decompress_allocate(zip_uint16_t method, int compression_flags, zip_error_t *error) {
+    return allocate(false, compression_flags, error, method);
+}
+
+
+static void
+deallocate(void *ud) {
+    struct ctx *ctx = (struct ctx *)ud;
+    free(ctx);
+}
+
+
+static zip_uint16_t
+general_purpose_bit_flags(void *ud) {
+    /* struct ctx *ctx = (struct ctx *)ud; */
+    return 0;
+}
+
+static int
+map_error(size_t ret) {
+    return 0;
+}
+
+
+static bool
+start(void *ud) {
+    struct ctx *ctx = (struct ctx *)ud;
+    ctx->in.src = NULL;
+    ctx->in.pos = 0;
+    ctx->in.size = 0;
+    ctx->out.dst = NULL;
+    ctx->out.pos = 0;
+    ctx->out.size = 0;
+    if (ctx->compress) {
+	ctx->zcstream = ZSTD_createCStream();
+    }
+    else {
+	ctx->zdstream = ZSTD_createDStream();
+    }
+
+    return true;
+}
+
+
+static bool
+end(void *ud) {
+    struct ctx *ctx = (struct ctx *)ud;
+    if (ctx->compress) {
+	ZSTD_freeCStream(ctx->zcstream);
+	ctx->zcstream = NULL;
+    }
+    else {
+	ZSTD_freeDStream(ctx->zdstream);
+	ctx->zdstream = NULL;
+    }
+
+    return true;
+}
+
+
+static bool
+input(void *ud, zip_uint8_t *data, zip_uint64_t length) {
+    struct ctx *ctx = (struct ctx *)ud;
+    if (ctx->in.pos != 0 && ctx->in.pos != ctx->in.size) {
+	zip_error_set(ctx->error, ZIP_ER_INVAL, 0);
+	return false;
+    }
+    ctx->in.src = (const void *)data;
+    ctx->in.size = length;
+    ctx->in.pos = 0;
+    return true;
+}
+
+
+static void
+end_of_input(void *ud) {
+    struct ctx *ctx = (struct ctx *)ud;
+
+    ctx->end_of_input = true;
+}
+
+
+static zip_compression_status_t
+process(void *ud, zip_uint8_t *data, zip_uint64_t *length) {
+    struct ctx *ctx = (struct ctx *)ud;
+
+    size_t ret;
+
+    ctx->out.dst = data;
+    ctx->out.pos = 0;
+    ctx->out.size = ZIP_MIN(UINT_MAX, *length);
+
+    if (ctx->compress) {
+	ret = ZSTD_compressStream(ctx->zcstream, &ctx->out, &ctx->in);
+    }
+    else {
+	ret = ZSTD_decompressStream(ctx->zdstream, &ctx->out, &ctx->in);
+    }
+    if (ZSTD_isError(ret)) {
+	zip_error_set(ctx->error, ZIP_ER_ZSTD, (int)ret);
+	return ZIP_COMPRESSION_ERROR;
+    }
+    *length = ctx->out.pos;
+    if (ctx->out.pos == 0) {
+	return ZIP_COMPRESSION_NEED_DATA;
+    }
+    if (ctx->in.size == 0) {
+	return ZIP_COMPRESSION_END;
+    }
+    return ZIP_COMPRESSION_OK;
+}
+
+/* clang-format off */
+
+zip_compression_algorithm_t zip_algorithm_zstd_compress = {
+    compress_allocate,
+    deallocate,
+    general_purpose_bit_flags,
+    20,
+    start,
+    end,
+    input,
+    end_of_input,
+    process
+};
+
+
+zip_compression_algorithm_t zip_algorithm_zstd_decompress = {
+    decompress_allocate,
+    deallocate,
+    general_purpose_bit_flags,
+    20,
+    start,
+    end,
+    input,
+    end_of_input,
+    process
+};
+
+/* clang-format on */

lib/zip_source_compress.c

@@ -75,6 +75,9 @@ static struct implementation implementations[] = {
     */
     {ZIP_CM_XZ, &zip_algorithm_xz_compress, &zip_algorithm_xz_decompress},
 #endif
+#if defined(HAVE_LIBZSTD)
+    {ZIP_CM_ZSTD, &zip_algorithm_zstd_compress, &zip_algorithm_zstd_decompress},
+#endif
 
 };
 

lib/zipint.h

@@ -153,6 +153,8 @@ extern zip_compression_algorithm_t zip_algorithm_deflate_compress;
 extern zip_compression_algorithm_t zip_algorithm_deflate_decompress;
 extern zip_compression_algorithm_t zip_algorithm_xz_compress;
 extern zip_compression_algorithm_t zip_algorithm_xz_decompress;
+extern zip_compression_algorithm_t zip_algorithm_zstd_compress;
+extern zip_compression_algorithm_t zip_algorithm_zstd_decompress;
 
 
 /* This API is not final yet, but we need it internally, so it's private for now. */

src/ziptool.c

@@ -652,6 +652,11 @@ get_compression_method(const char *arg) {
     else if (strcasecmp(arg, "xz") == 0)
       return ZIP_CM_XZ;
 
+#endif
+#if defined(HAVE_LIBZSTD)
+    else if (strcasecmp(arg, "zstd") == 0)
+      return ZIP_CM_ZSTD;
+
 #endif
     else if (strcasecmp(arg, "unknown") == 0)
 	return 100;