add zstd compression support, fix NeuroJSON#5

fangq · Sep 18, 2022 · 877f2d7 · 877f2d7
1 parent d62c9de
commit 877f2d7
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 12 deletions.
diff --git a/include/zmatlib.h b/include/zmatlib.h
@@ -63,15 +63,16 @@ extern "C"
  * 4: lzma
  * 5: lz4
  * 6: lz4hc
- * 7: blosc2blosclz
- * 8: blosc2lz4
- * 9: blosc2lz4hc
- * 10: blosc2zlib
- * 11: blosc2zstd
+ * 7: zstd
+ * 8: blosc2blosclz
+ * 9: blosc2lz4
+ * 10: blosc2lz4hc
+ * 11: blosc2zlib
+ * 12: blosc2zstd
  * -1: unknown
  */
 
-enum TZipMethod {zmZlib, zmGzip, zmBase64, zmLzip, zmLzma, zmLz4, zmLz4hc, zmBlosc2Blosclz, zmBlosc2Lz4, zmBlosc2Lz4hc, zmBlosc2Zlib, zmBlosc2Zstd, zmUnknown = -1};
+enum TZipMethod {zmZlib, zmGzip, zmBase64, zmLzip, zmLzma, zmLz4, zmLz4hc, zmZstd, zmBlosc2Blosclz, zmBlosc2Lz4, zmBlosc2Lz4hc, zmBlosc2Zlib, zmBlosc2Zstd, zmUnknown = -1};
 
 /**
  * @brief Main interface to perform compression/decompression

diff --git a/src/Makefile b/src/Makefile
@@ -13,6 +13,7 @@ LIBDIR  ?=$(ROOTDIR)/lib
 MKDIR      :=mkdir
 HAVE_LZMA  ?=yes
 HAVE_LZ4   ?=yes
+HAVE_ZSTD  ?=yes
 HAVE_BLOSC2?=yes
 
 MEX=mex
@@ -81,6 +82,10 @@ else
     INCLUDEDIRS+=-Ilz4
     FILES+= lz4/lz4 lz4/lz4hc
   endif
+  ifeq ($(HAVE_ZSTD),no)
+    INCLUDEDIRS+=-Iblosc2/internal-complibs/zstd-1.5.2
+    LINKOPT+=-Lblosc2/internal-complibs/zstd-1.5.2 -lzstd
+  endif
   LINKOPT+=-Lblosc2/lib -lblosc2 -Lblosc2/internal-complibs/zstd-1.5.2 -lzstd
   INCLUDEDIRS+=-Iblosc2/include
 endif
@@ -92,7 +97,12 @@ else
   FILES+= lz4/lz4 lz4/lz4hc
 endif
 
-
+ifeq ($(HAVE_ZSTD),no)
+  CFLAGS+=-DNO_ZSTD
+else
+  INCLUDEDIRS+=-Iblosc2/internal-complibs/zstd-1.5.2
+  LINKOPT+=-Lblosc2/internal-complibs/zstd-1.5.2 -lzstd
+endif
 
 
 ifeq ($(MAKECMDGOALS),lib)

diff --git a/src/zmat.cpp b/src/zmat.cpp
@@ -77,6 +77,9 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
         "lz4",
         "lz4hc",
 #endif
+#if !defined(NO_ZSTD)
+        "zstd",
+#endif
 #if !defined(NO_BLOSC2)
         "blosc2blosclz",
         "blosc2lz4",
@@ -99,6 +102,9 @@ void mexFunction(int nlhs, mxArray* plhs[], int nrhs, const mxArray* prhs[]) {
         zmLz4,
         zmLz4hc,
 #endif
+#if !defined(NO_ZSTD)
+        zmZstd,
+#endif
 #if !defined(NO_BLOSC2)
         zmBlosc2Blosclz,
         zmBlosc2Lz4,

diff --git a/src/zmatlib.c b/src/zmatlib.c
@@ -65,6 +65,11 @@
     #include "blosc2.h"
 #endif
 
+#ifndef NO_ZSTD
+    #include "zstd.h"
+#endif
+
+
 #ifndef NO_LZMA
 /**
  * @brief Easylzma interface to perform compression
@@ -119,6 +124,7 @@ const char* zmat_errcode[] = {
     "lz4 error, see info.status for error flag, often a result of mismatch in compression method",/*-6*/
     "unsupported blosc2 codec",/*-7*/
     "blosc2 error, see info.status for error flag, often a result of mismatch in compression method",/*-8*/
+    "zstd error, see info.status for error flag, often a result of mismatch in compression method",/*-9*/
     "unsupported method" /*-999*/
 };
 
@@ -141,9 +147,9 @@ char* zmat_error(int id) {
  *
  * @param[in] inputsize: input stream buffer length
  * @param[in] inputstr: input stream buffer pointer
- * @param[in] outputsize: output stream buffer length
- * @param[in] outputbuf: output stream buffer pointer
- * @param[in] ret: encoder/decoder specific detailed error code (if error occurs)
+ * @param[in, out] outputsize: output stream buffer length
+ * @param[in, out] outputbuf: output stream buffer pointer
+ * @param[out] ret: encoder/decoder specific detailed error code (if error occurs)
  * @param[in] iscompress: 0: decompression, 1: use default compression level;
  *             negative interger: set compression level (-1, less, to -9, more compression)
  * @return return the coarse grained zmat error code; detailed error code is in ret.
@@ -256,6 +262,30 @@ int zmat_run(const size_t inputsize, unsigned char* inputstr, size_t* outputsize
                 return -6;
             }
 
+#endif
+#ifndef NO_ZSTD
+        } else if (zipid == zmZstd) {
+            /**
+              * zstd compression
+              */
+            *outputsize = ZSTD_compressBound(inputsize);
+
+            if (!(*outputbuf = (unsigned char*)malloc(*outputsize))) {
+                return -5;
+            }
+
+            *ret = ZSTD_compress((char*)(*outputbuf), *outputsize, (const char*)inputstr, inputsize, (clevel > 0) ? 8 : (-clevel));
+
+            if (ZSTD_isError(*ret)) {
+                return -9;
+            }
+
+            *outputsize = *ret;
+
+            if (!(*outputbuf = (unsigned char*)realloc(*outputbuf, *outputsize))) {
+                return -5;
+            }
+
 #endif
 #ifndef NO_BLOSC2
         } else if (zipid >= zmBlosc2Blosclz || zipid <= zmBlosc2Zstd) {
@@ -278,12 +308,12 @@ int zmat_run(const size_t inputsize, unsigned char* inputstr, size_t* outputsize
 
             *ret = blosc1_compress((clevel > 0) ? 5 : (-clevel), shuffle, typesize, inputsize, (const void*)inputstr, (void*)(*outputbuf), *outputsize);
 
-            *outputsize = *ret;
-
             if (*ret < 0) {
                 return -8;
             }
 
+            *outputsize = *ret;
+
             if (!(*outputbuf = (unsigned char*)realloc(*outputbuf, *outputsize))) {
                 return -5;
             }
@@ -384,6 +414,27 @@ int zmat_run(const size_t inputsize, unsigned char* inputstr, size_t* outputsize
                 return -6;
             }
 
+#endif
+#ifndef NO_ZSTD
+        } else if (zipid == zmZstd) {
+            /**
+              * zstd decompression
+              */
+            *outputsize = ZSTD_decompressBound(inputstr, inputsize);
+
+            if (*outputsize == ZSTD_CONTENTSIZE_ERROR || !(*outputbuf = (unsigned char*)malloc(*outputsize))) {
+                *ret = -5;
+                return *ret;
+            }
+
+            *ret = ZSTD_decompress((void*)(*outputbuf), *outputsize, (const void*)inputstr, inputsize);
+
+            *outputsize = *ret;
+
+            if (ZSTD_isError(*ret)) {
+                return -6;
+            }
+
 #endif
 #ifndef NO_BLOSC2
         } else if (zipid >= zmBlosc2Blosclz || zipid <= zmBlosc2Zstd) {

diff --git a/zmat.m b/zmat.m
@@ -3,6 +3,7 @@
 % output=zmat(input)
 %    or
 % [output, info]=zmat(input, iscompress, method)
+% [output, info]=zmat(input, iscompress, method, options ...)
 % output=zmat(input, info)
 %
 % A portable data compression/decompression toolbox for MATLAB/GNU Octave
@@ -33,7 +34,17 @@
 %             'lzma': lzma formatted data compression
 %             'lz4':  lz4 formatted data compression
 %             'lz4hc':lz4hc (LZ4 with high-compression ratio) formatted data compression
+%             'zstd':  zstd formatted data compression
+%             'blosc2blosclz':  blosc2 meta-compressor with blosclz compression
+%             'blosc2lz4':  blosc2 meta-compressor with lz4 compression
+%             'blosc2lz4hc':  blosc2 meta-compressor with lz4hc compression
+%             'blosc2zlib:  blosc2 meta-compressor with zlib/zip compression
+%             'blosc2zstd':  blosc2 meta-compressor with zstd compression
 %             'base64': encode or decode use base64 format
+%     options: a series of ('name', value) pairs, supported options include
+%             'nthread': followed by an integer specifying number of threads for blosc2 meta-compressors
+%             'typesize': followed by an integer specifying the number of bytes per data element (used for shuffle)
+%             'shuffle': shuffle methods in blosc2 meta-compressor, 0 disable, 1, byte-shuffle
 %
 % output:
 %      output: a uint8 row vector, storing the compressed or decompressed data;
@@ -136,6 +147,8 @@
     varargout{1} = reshape(varargout{1}, inputinfo.size);
 end
 
+%--------------------------------------------------------------------------------
+
 function value=getoption(key, default, opt)
 value=default;
 if(isfield(opt, key))