Merge pull request #354 from sebres/zstd-like-zstdcli--sizehint-checksum

Make compression of zstd archive type more similar to zstdcli
mcmilk · Sep 17, 2023 · ce27b4a · ce27b4a
2 parents 62bfad5 + 1eca571
commit ce27b4a
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 4 deletions.
diff --git a/C/zstdmt/brotli-mt_compress.c b/C/zstdmt/brotli-mt_compress.c
@@ -383,7 +383,7 @@ static size_t st_compress(void *arg)
 	  /* 0, or not specified by user; could be chosen by compressor. */
 	  uint32_t lgwin = 24 /* DEFAULT_LGWIN */;
 	  /* Use file size to limit lgwin. */
-	  if (ctx->unpackSize >= 0) {
+	  if (ctx->unpackSize >= 0 && ctx->unpackSize != (uint64_t)(int64_t)-1) {
 	    lgwin = BROTLI_MIN_WINDOW_BITS;
 	    while (BROTLI_MAX_BACKWARD_LIMIT(lgwin) <
 	           (uint64_t)ctx->unpackSize) {
@@ -393,7 +393,7 @@ static size_t st_compress(void *arg)
 	  }
 	  BrotliEncoderSetParameter(state, BROTLI_PARAM_LGWIN, lgwin);
 	}
-	if (ctx->unpackSize > 0) {
+	if (ctx->unpackSize > 0 && ctx->unpackSize != (uint64_t)(int64_t)-1) {
 		uint32_t size_hint = ctx->unpackSize < (1 << 30) ?
 	    (uint32_t)ctx->unpackSize : (1u << 30);
 		BrotliEncoderSetParameter(state, BROTLI_PARAM_SIZE_HINT, size_hint);

diff --git a/CPP/7zip/Archive/ZstdHandler.cpp b/CPP/7zip/Archive/ZstdHandler.cpp
@@ -286,6 +286,10 @@ static HRESULT UpdateArchive(
   CMyComPtr<ICompressProgressInfo> localProgress = localProgressSpec;
   localProgressSpec->Init(updateCallback, true);
   NCompress::NZSTD::CEncoder *encoderSpec = new NCompress::NZSTD::CEncoder;
+  // by zstd archive type store dictID and checksum (similar to zstd client)
+  encoderSpec->dictIDFlag = 1;
+  encoderSpec->checksumFlag = 1;
+  encoderSpec->unpackSize = unpackSize;
   CMyComPtr<ICompressCoder> encoder = encoderSpec;
   RINOK(props.SetCoderProps(encoderSpec, NULL));
   RINOK(encoder->Code(fileInStream, outStream, NULL, NULL, localProgress));

diff --git a/CPP/7zip/Compress/BrotliEncoder.cpp b/CPP/7zip/Compress/BrotliEncoder.cpp
@@ -15,7 +15,8 @@ CEncoder::CEncoder():
   _numThreads(NWindows::NSystem::GetNumberOfProcessors()),
   _Long(-1),
   _WindowLog(-1),
-  _ctx(NULL)
+  _ctx(NULL),
+  unpackSize(0)
 {
   _props.clear();
 }

diff --git a/CPP/7zip/Compress/ZstdEncoder.cpp b/CPP/7zip/Compress/ZstdEncoder.cpp
@@ -30,7 +30,10 @@ CEncoder::CEncoder():
   _LdmHashLog(-1),
   _LdmMinMatch(-1),
   _LdmBucketSizeLog(-1),
-  _LdmHashRateLog(-1)
+  _LdmHashRateLog(-1),
+  dictIDFlag(-1),
+  checksumFlag(-1),
+  unpackSize(0)
 {
   _props.clear();
 }
@@ -251,6 +254,20 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
     err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_contentSizeFlag, 1);
     if (ZSTD_isError(err)) return E_INVALIDARG;
 
+    if (dictIDFlag != -1) {
+      err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_dictIDFlag, dictIDFlag);
+      if (ZSTD_isError(err)) return E_INVALIDARG;
+    }
+    if (checksumFlag != -1) {
+      err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_checksumFlag, checksumFlag);
+      if (ZSTD_isError(err)) return E_INVALIDARG;
+    }
+
+    if (unpackSize && unpackSize != (UInt64)(Int64)-1) { // size is known
+      err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_srcSizeHint, (int)(unpackSize <= INT_MAX ? unpackSize : INT_MAX));
+      if (ZSTD_isError(err)) return E_INVALIDARG;
+    }
+
     /* enable ldm for large windowlog values */
     if (_WindowLog > 27 && _Long == 0)
       _Long = 1;
@@ -320,6 +337,12 @@ STDMETHODIMP CEncoder::Code(ISequentialInStream *inStream,
       err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_ldmHashRateLog, _LdmHashRateLog);
       if (ZSTD_isError(err)) return E_INVALIDARG;
     }
+
+    //err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_literalCompressionMode, (int)ZSTD_ps_auto);
+    //if (ZSTD_isError(err)) return E_INVALIDARG;
+
+    //err = ZSTD_CCtx_setParameter(_ctx, ZSTD_c_enableDedicatedDictSearch, 1);
+    //if (ZSTD_isError(err)) return E_INVALIDARG;
   }
 
   for (;;) {

diff --git a/CPP/7zip/Compress/ZstdEncoder.h b/CPP/7zip/Compress/ZstdEncoder.h
@@ -67,6 +67,11 @@ class CEncoder:
   Int32 _LdmHashRateLog;
 
 public:
+
+  int dictIDFlag;
+  int checksumFlag;
+  UInt64 unpackSize;
+
   MY_QUERYINTERFACE_BEGIN2(ICompressCoder)
   MY_QUERYINTERFACE_ENTRY(ICompressSetCoderMt)
   MY_QUERYINTERFACE_ENTRY(ICompressSetCoderProperties)