diff --git a/benchmark/zlib/creation.js b/benchmark/zlib/creation.js index 90b22780d2d312..41b1e4917a67bb 100644 --- a/benchmark/zlib/creation.js +++ b/benchmark/zlib/creation.js @@ -5,7 +5,7 @@ const zlib = require('zlib'); const bench = common.createBenchmark(main, { type: [ 'Deflate', 'DeflateRaw', 'Inflate', 'InflateRaw', 'Gzip', 'Gunzip', 'Unzip', - 'BrotliCompress', 'BrotliDecompress', + 'BrotliCompress', 'BrotliDecompress', 'ZstdCompress', 'ZstdDecompress', ], options: ['true', 'false'], n: [5e5], diff --git a/benchmark/zlib/pipe.js b/benchmark/zlib/pipe.js index a9c86e3de660a9..5a21c3ff417084 100644 --- a/benchmark/zlib/pipe.js +++ b/benchmark/zlib/pipe.js @@ -7,7 +7,7 @@ const bench = common.createBenchmark(main, { inputLen: [1024], duration: [5], type: ['string', 'buffer'], - algorithm: ['gzip', 'brotli'], + algorithm: ['gzip', 'brotli', 'zstd'], }, { test: { inputLen: 1024, @@ -15,14 +15,19 @@ const bench = common.createBenchmark(main, { }, }); +const algorithms = { + 'gzip': [zlib.createGzip, zlib.createGunzip], + 'brotli': [zlib.createBrotliCompress, zlib.createBrotliDecompress], + 'zstd': [zlib.createZstdCompress, zlib.createZstdDecompress], +}; + function main({ inputLen, duration, type, algorithm }) { const buffer = Buffer.alloc(inputLen, fs.readFileSync(__filename)); const chunk = type === 'buffer' ? buffer : buffer.toString('utf8'); - const input = algorithm === 'gzip' ? - zlib.createGzip() : zlib.createBrotliCompress(); - const output = algorithm === 'gzip' ? - zlib.createGunzip() : zlib.createBrotliDecompress(); + const [createCompress, createUncompress] = algorithms[algorithm]; + const input = createCompress(); + const output = createUncompress(); let readFromOutput = 0; input.pipe(output); diff --git a/doc/api/errors.md b/doc/api/errors.md index 558bc46088adef..b9cfc10ef12414 100644 --- a/doc/api/errors.md +++ b/doc/api/errors.md @@ -3800,6 +3800,12 @@ removed: v10.0.0 Used when an attempt is made to use a `zlib` object after it has already been closed. + + +### `ERR_ZSTD_INVALID_PARAM` + +An invalid parameter key was passed during construction of a Zstd stream. + ### `ERR_CPU_USAGE` diff --git a/doc/api/zlib.md b/doc/api/zlib.md index b54fcbaeefe5c2..4ee28f503c78a2 100644 --- a/doc/api/zlib.md +++ b/doc/api/zlib.md @@ -124,8 +124,8 @@ operations be cached to avoid duplication of effort. ## Compressing HTTP requests and responses -The `node:zlib` module can be used to implement support for the `gzip`, `deflate` -and `br` content-encoding mechanisms defined by +The `node:zlib` module can be used to implement support for the `gzip`, `deflate`, +`br` and `zstd` content-encoding mechanisms defined by [HTTP](https://tools.ietf.org/html/rfc7230#section-4.2). The HTTP [`Accept-Encoding`][] header is used within an HTTP request to identify @@ -148,7 +148,7 @@ const { pipeline } = require('node:stream'); const request = http.get({ host: 'example.com', path: '/', port: 80, - headers: { 'Accept-Encoding': 'br,gzip,deflate' } }); + headers: { 'Accept-Encoding': 'br,gzip,deflate,zstd' } }); request.on('response', (response) => { const output = fs.createWriteStream('example.com_index.html'); @@ -170,6 +170,9 @@ request.on('response', (response) => { case 'deflate': pipeline(response, zlib.createInflate(), output, onError); break; + case 'zstd': + pipeline(response, zlib.createZstdDecompress(), output, onError); + break; default: pipeline(response, output, onError); break; @@ -218,6 +221,9 @@ http.createServer((request, response) => { } else if (/\bbr\b/.test(acceptEncoding)) { response.writeHead(200, { 'Content-Encoding': 'br' }); pipeline(raw, zlib.createBrotliCompress(), response, onError); + } else if (/\bzstd\b/.test(acceptEncoding)) { + response.writeHead(200, { 'Content-Encoding': 'zstd' }); + pipeline(raw, zlib.createZstdCompress(), response, onError); } else { response.writeHead(200, {}); pipeline(raw, response, onError); @@ -238,6 +244,7 @@ const buffer = Buffer.from('eJzT0yMA', 'base64'); zlib.unzip( buffer, // For Brotli, the equivalent is zlib.constants.BROTLI_OPERATION_FLUSH. + // For Zstd, the equivalent is zlib.constants.ZSTD_e_flush. { finishFlush: zlib.constants.Z_SYNC_FLUSH }, (err, buffer) => { if (err) { @@ -309,6 +316,16 @@ these options have different ranges than the zlib ones: See [below][Brotli parameters] for more details on Brotli-specific options. +### For Zstd-based streams + +There are equivalents to the zlib options for Zstd-based streams, although +these options have different ranges than the zlib ones: + +* zlib's `level` option matches Zstd's `ZSTD_c_compressionLevel` option. +* zlib's `windowBits` option matches Zstd's `ZSTD_c_windowLog` option. + +See [below][Zstd parameters] for more details on Zstd-specific options. + ## Flushing Calling [`.flush()`][] on a compression stream will make `zlib` return as much @@ -487,6 +504,50 @@ These advanced options are available for controlling decompression: * Boolean flag enabling “Large Window Brotli” mode (not compatible with the Brotli format as standardized in [RFC 7932][]). +### Zstd constants + + + +There are several options and other constants available for Zstd-based +streams: + +#### Flush operations + +The following values are valid flush operations for Zstd-based streams: + +* `zlib.constants.ZSTD_e_continue` (default for all operations) +* `zlib.constants.ZSTD_e_flush` (default when calling `.flush()`) +* `zlib.constants.ZSTD_e_end` (default for the last chunk) + +#### Compressor options + +There are several options that can be set on Zstd encoders, affecting +compression efficiency and speed. Both the keys and the values can be accessed +as properties of the `zlib.constants` object. + +The most important options are: + +* `ZSTD_c_compressionLevel` + * Set compression parameters according to pre-defined cLevel table. Default + level is ZSTD\_CLEVEL\_DEFAULT==3. + +#### Pledged Source Size + +It's possible to specify the expected total size of the uncompressed input via +`opts.pledgedSrcSize`. If the size doesn't match at the end of the input, +compression will fail with the code `ZSTD_error_srcSize_wrong`. + +#### Decompressor options + +These advanced options are available for controlling decompression: + +* `ZSTD_d_windowLogMax` + * Select a size limit (in power of 2) beyond which the streaming API will + refuse to allocate memory buffer in order to protect the host from + unreasonable memory requirements. + ## Class: `Options` + + + +Each Zstd-based class takes an `options` object. All options are optional. + +* `flush` {integer} **Default:** `zlib.constants.ZSTD_e_continue` +* `finishFlush` {integer} **Default:** `zlib.constants.ZSTD_e_end` +* `chunkSize` {integer} **Default:** `16 * 1024` +* `params` {Object} Key-value object containing indexed [Zstd parameters][]. +* `maxOutputLength` {integer} Limits output size when using + [convenience methods][]. **Default:** [`buffer.kMaxLength`][] + +For example: + +```js +const stream = zlib.createZstdCompress({ + chunkSize: 32 * 1024, + params: { + [zlib.constants.ZSTD_c_compressionLevel]: 10, + [zlib.constants.ZSTD_c_checksumFlag]: 1, + }, +}); +``` + +## Class: `zlib.ZstdCompress` + + + +Compress data using the Zstd algorithm. + +## Class: `zlib.ZstdDecompress` + + + +Decompress data using the Zstd algorithm. + ### `zlib.bytesRead` + +* `options` {zstd options} + +Creates and returns a new [`ZstdCompress`][] object. + +## `zlib.createZstdDecompress([options])` + + + +* `options` {zstd options} + +Creates and returns a new [`ZstdDecompress`][] object. + ## Convenience methods @@ -1283,11 +1409,54 @@ changes: Decompress a chunk of data with [`Unzip`][]. +### `zlib.zstdCompress(buffer[, options], callback)` + + + +* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string} +* `options` {zstd options} +* `callback` {Function} + +### `zlib.zstdCompressSync(buffer[, options])` + + + +* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string} +* `options` {zstd options} + +Compress a chunk of data with [`ZstdCompress`][]. + +### `zlib.zstdDecompress(buffer[, options], callback)` + + + +* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string} +* `options` {zstd options} +* `callback` {Function} + +### `zlib.zstdDecompressSync(buffer[, options])` + + + +* `buffer` {Buffer|TypedArray|DataView|ArrayBuffer|string} +* `options` {zstd options} + +Decompress a chunk of data with [`ZstdDecompress`][]. + [Brotli parameters]: #brotli-constants [Cyclic redundancy check]: https://en.wikipedia.org/wiki/Cyclic_redundancy_check [Memory usage tuning]: #memory-usage-tuning [RFC 7932]: https://www.rfc-editor.org/rfc/rfc7932.txt [Streams API]: stream.md +[Zstd parameters]: #zstd-constants [`.flush()`]: #zlibflushkind-callback [`Accept-Encoding`]: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.3 [`ArrayBuffer`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/ArrayBuffer @@ -1304,6 +1473,8 @@ Decompress a chunk of data with [`Unzip`][]. [`Inflate`]: #class-zlibinflate [`TypedArray`]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/TypedArray [`Unzip`]: #class-zlibunzip +[`ZstdCompress`]: #class-zlibzstdcompress +[`ZstdDecompress`]: #class-zlibzstddecompress [`buffer.kMaxLength`]: buffer.md#bufferkmaxlength [`deflateInit2` and `inflateInit2`]: https://zlib.net/manual.html#Advanced [`stream.Transform`]: stream.md#class-streamtransform diff --git a/lib/internal/errors.js b/lib/internal/errors.js index 95acfef204734b..309602f3bff48c 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -1887,3 +1887,4 @@ E('ERR_WORKER_UNSERIALIZABLE_ERROR', E('ERR_WORKER_UNSUPPORTED_OPERATION', '%s is not supported in workers', TypeError); E('ERR_ZLIB_INITIALIZATION_FAILED', 'Initialization failed', Error); +E('ERR_ZSTD_INVALID_PARAM', '%s is not a valid zstd parameter', RangeError); diff --git a/lib/zlib.js b/lib/zlib.js index a2c092f1037261..61c5d6d6a8f702 100644 --- a/lib/zlib.js +++ b/lib/zlib.js @@ -48,6 +48,7 @@ const { ERR_INVALID_ARG_TYPE, ERR_OUT_OF_RANGE, ERR_ZLIB_INITIALIZATION_FAILED, + ERR_ZSTD_INVALID_PARAM, }, genericNodeError, } = require('internal/errors'); @@ -89,9 +90,12 @@ const { // Node's compression stream modes (node_zlib_mode) DEFLATE, DEFLATERAW, INFLATE, INFLATERAW, GZIP, GUNZIP, UNZIP, BROTLI_DECODE, BROTLI_ENCODE, + ZSTD_COMPRESS, ZSTD_DECOMPRESS, // Brotli operations (~flush levels) BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_FLUSH, BROTLI_OPERATION_FINISH, BROTLI_OPERATION_EMIT_METADATA, + // Zstd end directives (~flush levels) + ZSTD_e_continue, ZSTD_e_flush, ZSTD_e_end, } = constants; // Translation table for return codes. @@ -197,9 +201,11 @@ function zlibOnError(message, errno, code) { const FLUSH_BOUND = [ [ Z_NO_FLUSH, Z_BLOCK ], [ BROTLI_OPERATION_PROCESS, BROTLI_OPERATION_EMIT_METADATA ], + [ ZSTD_e_continue, ZSTD_e_end ], ]; const FLUSH_BOUND_IDX_NORMAL = 0; const FLUSH_BOUND_IDX_BROTLI = 1; +const FLUSH_BOUND_IDX_ZSTD = 2; // The base class for all Zlib-style streams. function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) { @@ -208,13 +214,15 @@ function ZlibBase(opts, mode, handle, { flush, finishFlush, fullFlush }) { // The ZlibBase class is not exported to user land, the mode should only be // passed in by us. assert(typeof mode === 'number'); - assert(mode >= DEFLATE && mode <= BROTLI_ENCODE); + assert(mode >= DEFLATE && mode <= ZSTD_DECOMPRESS); let flushBoundIdx; - if (mode !== BROTLI_ENCODE && mode !== BROTLI_DECODE) { - flushBoundIdx = FLUSH_BOUND_IDX_NORMAL; - } else { + if (mode === BROTLI_ENCODE || mode === BROTLI_DECODE) { flushBoundIdx = FLUSH_BOUND_IDX_BROTLI; + } else if (mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS) { + flushBoundIdx = FLUSH_BOUND_IDX_ZSTD; + } else { + flushBoundIdx = FLUSH_BOUND_IDX_NORMAL; } if (opts) { @@ -847,6 +855,91 @@ ObjectSetPrototypeOf(BrotliDecompress.prototype, Brotli.prototype); ObjectSetPrototypeOf(BrotliDecompress, Brotli); +const zstdDefaultOpts = { + flush: ZSTD_e_continue, + finishFlush: ZSTD_e_end, + fullFlush: ZSTD_e_flush, +}; +function Zstd(opts, mode, initParamsArray, maxParam) { + assert(mode === ZSTD_COMPRESS || mode === ZSTD_DECOMPRESS); + + TypedArrayPrototypeFill(initParamsArray, -1); + if (opts?.params) { + ArrayPrototypeForEach(ObjectKeys(opts.params), (origKey) => { + const key = +origKey; + if (NumberIsNaN(key) || key < 0 || key > maxParam || + (initParamsArray[key] | 0) !== -1) { + throw new ERR_ZSTD_INVALID_PARAM(origKey); + } + + const value = opts.params[origKey]; + if (typeof value !== 'number' && typeof value !== 'boolean') { + throw new ERR_INVALID_ARG_TYPE('options.params[key]', + 'number', opts.params[origKey]); + } + initParamsArray[key] = value; + }); + } + + const handle = mode === ZSTD_COMPRESS ? + new binding.ZstdCompress() : new binding.ZstdDecompress(); + + const pledgedSrcSize = opts?.pledgedSrcSize ?? undefined; + + this._writeState = new Uint32Array(2); + if (!handle.init(initParamsArray, + pledgedSrcSize, + this._writeState, + processCallback)) { + throw new ERR_ZLIB_INITIALIZATION_FAILED(); + } + + ReflectApply(ZlibBase, this, [opts, mode, handle, zstdDefaultOpts]); +} +ObjectSetPrototypeOf(Zstd.prototype, ZlibBase.prototype); +ObjectSetPrototypeOf(Zstd, ZlibBase); + + +const kMaxZstdCParam = MathMaxApply(ArrayPrototypeMap( + ObjectKeys(constants), + (key) => (StringPrototypeStartsWith(key, 'ZSTD_c_') ? + constants[key] : + 0), +)); + +const zstdInitCParamsArray = new Uint32Array(kMaxZstdCParam + 1); + +function ZstdCompress(opts) { + if (!(this instanceof ZstdCompress)) + return new ZstdCompress(opts); + + ReflectApply(Zstd, this, + [opts, ZSTD_COMPRESS, zstdInitCParamsArray, kMaxZstdCParam]); +} +ObjectSetPrototypeOf(ZstdCompress.prototype, Zstd.prototype); +ObjectSetPrototypeOf(ZstdCompress, Zstd); + + +const kMaxZstdDParam = MathMaxApply(ArrayPrototypeMap( + ObjectKeys(constants), + (key) => (StringPrototypeStartsWith(key, 'ZSTD_d_') ? + constants[key] : + 0), +)); + +const zstdInitDParamsArray = new Uint32Array(kMaxZstdDParam + 1); + +function ZstdDecompress(opts) { + if (!(this instanceof ZstdDecompress)) + return new ZstdDecompress(opts); + + ReflectApply(Zstd, this, + [opts, ZSTD_DECOMPRESS, zstdInitDParamsArray, kMaxZstdDParam]); +} +ObjectSetPrototypeOf(ZstdDecompress.prototype, Zstd.prototype); +ObjectSetPrototypeOf(ZstdDecompress, Zstd); + + function createProperty(ctor) { return { __proto__: null, @@ -885,6 +978,8 @@ module.exports = { Unzip, BrotliCompress, BrotliDecompress, + ZstdCompress, + ZstdDecompress, // Convenience methods. // compress/decompress a string or buffer in one step. @@ -906,6 +1001,10 @@ module.exports = { brotliCompressSync: createConvenienceMethod(BrotliCompress, true), brotliDecompress: createConvenienceMethod(BrotliDecompress, false), brotliDecompressSync: createConvenienceMethod(BrotliDecompress, true), + zstdCompress: createConvenienceMethod(ZstdCompress, false), + zstdCompressSync: createConvenienceMethod(ZstdCompress, true), + zstdDecompress: createConvenienceMethod(ZstdDecompress, false), + zstdDecompressSync: createConvenienceMethod(ZstdDecompress, true), }; ObjectDefineProperties(module.exports, { @@ -918,6 +1017,8 @@ ObjectDefineProperties(module.exports, { createUnzip: createProperty(Unzip), createBrotliCompress: createProperty(BrotliCompress), createBrotliDecompress: createProperty(BrotliDecompress), + createZstdCompress: createProperty(ZstdCompress), + createZstdDecompress: createProperty(ZstdDecompress), constants: { __proto__: null, configurable: false, diff --git a/src/node_zlib.cc b/src/node_zlib.cc index 3ff3ac13112330..4ebd6f746ff6d3 100644 --- a/src/node_zlib.cc +++ b/src/node_zlib.cc @@ -31,9 +31,11 @@ #include "v8.h" -#include "brotli/encode.h" #include "brotli/decode.h" +#include "brotli/encode.h" #include "zlib.h" +#include "zstd.h" +#include "zstd_errors.h" #include @@ -94,6 +96,44 @@ inline const char* ZlibStrerror(int err) { return "Z_UNKNOWN_ERROR"; } +#define ZSTD_ERROR_CODES(V) \ + V(ZSTD_error_no_error) \ + V(ZSTD_error_GENERIC) \ + V(ZSTD_error_prefix_unknown) \ + V(ZSTD_error_version_unsupported) \ + V(ZSTD_error_frameParameter_unsupported) \ + V(ZSTD_error_frameParameter_windowTooLarge) \ + V(ZSTD_error_corruption_detected) \ + V(ZSTD_error_checksum_wrong) \ + V(ZSTD_error_literals_headerWrong) \ + V(ZSTD_error_dictionary_corrupted) \ + V(ZSTD_error_dictionary_wrong) \ + V(ZSTD_error_dictionaryCreation_failed) \ + V(ZSTD_error_parameter_unsupported) \ + V(ZSTD_error_parameter_combination_unsupported) \ + V(ZSTD_error_parameter_outOfBound) \ + V(ZSTD_error_tableLog_tooLarge) \ + V(ZSTD_error_maxSymbolValue_tooLarge) \ + V(ZSTD_error_maxSymbolValue_tooSmall) \ + V(ZSTD_error_stabilityCondition_notRespected) \ + V(ZSTD_error_stage_wrong) \ + V(ZSTD_error_init_missing) \ + V(ZSTD_error_memory_allocation) \ + V(ZSTD_error_workSpace_tooSmall) \ + V(ZSTD_error_dstSize_tooSmall) \ + V(ZSTD_error_srcSize_wrong) \ + V(ZSTD_error_dstBuffer_null) \ + V(ZSTD_error_noForwardProgress_destFull) \ + V(ZSTD_error_noForwardProgress_inputEmpty) + +inline const char* ZstdStrerror(int err) { +#define V(code) \ + if (err == code) return #code; + ZSTD_ERROR_CODES(V) +#undef V + return "ZSTD_error_GENERIC"; +} + enum node_zlib_mode { NONE, DEFLATE, @@ -104,7 +144,9 @@ enum node_zlib_mode { INFLATERAW, UNZIP, BROTLI_DECODE, - BROTLI_ENCODE + BROTLI_ENCODE, + ZSTD_COMPRESS, + ZSTD_DECOMPRESS }; constexpr uint8_t GZIP_HEADER_ID1 = 0x1f; @@ -248,6 +290,79 @@ class BrotliDecoderContext final : public BrotliContext { DeleteFnPtr state_; }; +class ZstdContext : public MemoryRetainer { + public: + ZstdContext() = default; + + // Streaming-related, should be available for all compression libraries: + void Close(); + void SetBuffers(const char* in, uint32_t in_len, char* out, uint32_t out_len); + void SetFlush(int flush); + void GetAfterWriteOffsets(uint32_t* avail_in, uint32_t* avail_out) const; + CompressionError GetErrorInfo() const; + + ZstdContext(const ZstdContext&) = delete; + ZstdContext& operator=(const ZstdContext&) = delete; + + protected: + ZSTD_EndDirective flush_ = ZSTD_e_continue; + + ZSTD_inBuffer input_ = {nullptr, 0, 0}; + ZSTD_outBuffer output_ = {nullptr, 0, 0}; + + ZSTD_ErrorCode error_ = ZSTD_error_no_error; + std::string error_string_; + std::string error_code_string_; +}; + +class ZstdCompressContext final : public ZstdContext { + public: + ZstdCompressContext() = default; + + // Streaming-related, should be available for all compression libraries: + void DoThreadPoolWork(); + CompressionError ResetStream(); + + // Zstd specific: + CompressionError Init(uint64_t pledged_src_size); + CompressionError SetParameter(int key, int value); + + // Wrap ZSTD_freeCCtx to remove the return type. + static void FreeCCtx(ZSTD_CCtx* cctx) { ZSTD_freeCCtx(cctx); } + + SET_MEMORY_INFO_NAME(ZstdCompressContext) + SET_SELF_SIZE(ZstdCompressContext) + SET_NO_MEMORY_INFO() + + private: + DeleteFnPtr cctx_; + + uint64_t pledged_src_size_ = ZSTD_CONTENTSIZE_UNKNOWN; +}; + +class ZstdDecompressContext final : public ZstdContext { + public: + ZstdDecompressContext() = default; + + // Streaming-related, should be available for all compression libraries: + void DoThreadPoolWork(); + CompressionError ResetStream(); + + // Zstd specific: + CompressionError Init(uint64_t pledged_src_size); + CompressionError SetParameter(int key, int value); + + // Wrap ZSTD_freeDCtx to remove the return type. + static void FreeDCtx(ZSTD_DCtx* dctx) { ZSTD_freeDCtx(dctx); } + + SET_MEMORY_INFO_NAME(ZstdDecompressContext) + SET_SELF_SIZE(ZstdDecompressContext) + SET_NO_MEMORY_INFO() + + private: + DeleteFnPtr dctx_; +}; + template class CompressionStream : public AsyncWrap, public ThreadPoolWork { public: @@ -730,6 +845,92 @@ class BrotliCompressionStream final : using BrotliEncoderStream = BrotliCompressionStream; using BrotliDecoderStream = BrotliCompressionStream; +template +class ZstdStream final : public CompressionStream { + public: + ZstdStream(Environment* env, Local wrap) + : CompressionStream(env, wrap) {} + + inline CompressionContext* context() { + return this->CompressionStream::context(); + } + typedef typename CompressionStream::AllocScope AllocScope; + + static void New(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + new ZstdStream(env, args.This()); + } + + static void Init(const FunctionCallbackInfo& args) { + Environment* env = Environment::GetCurrent(args); + Local context = env->context(); + + CHECK(args.Length() == 4 && + "init(params, pledgedSrcSize, writeResult, writeCallback)"); + ZstdStream* wrap; + ASSIGN_OR_RETURN_UNWRAP(&wrap, args.Holder()); + + CHECK(args[2]->IsUint32Array()); + uint32_t* write_result = reinterpret_cast(Buffer::Data(args[2])); + + CHECK(args[3]->IsFunction()); + Local write_js_callback = args[3].As(); + wrap->InitStream(write_result, write_js_callback); + + uint64_t pledged_src_size = ZSTD_CONTENTSIZE_UNKNOWN; + if (args[1]->IsNumber()) { + int64_t signed_pledged_src_size; + if (!args[1]->IntegerValue(context).To(&signed_pledged_src_size)) { + args.GetReturnValue().Set(false); + return; + } + if (signed_pledged_src_size < 0) { + args.GetReturnValue().Set(false); + return; + } + pledged_src_size = signed_pledged_src_size; + } + + AllocScope alloc_scope(wrap); + CompressionError err = wrap->context()->Init(pledged_src_size); + if (err.IsError()) { + wrap->EmitError(err); + args.GetReturnValue().Set(false); + return; + } + + CHECK(args[0]->IsUint32Array()); + const uint32_t* data = reinterpret_cast(Buffer::Data(args[0])); + size_t len = args[0].As()->Length(); + + for (int i = 0; static_cast(i) < len; i++) { + if (data[i] == static_cast(-1)) continue; + + CompressionError err = wrap->context()->SetParameter(i, data[i]); + if (err.IsError()) { + wrap->EmitError(err); + args.GetReturnValue().Set(false); + return; + } + } + + args.GetReturnValue().Set(true); + } + + static void Params(const FunctionCallbackInfo& args) { + // Currently a no-op, and not accessed from JS land. + // At some point zstd may support changing parameters on the fly, + // in which case we can implement this and a JS equivalent similar to + // the zlib Params() function. + } + + SET_MEMORY_INFO_NAME(ZstdStream) + SET_SELF_SIZE(ZstdStream) +}; + +using ZstdCompressStream = ZstdStream; +using ZstdDecompressStream = ZstdStream; + void ZlibContext::Close() { { Mutex::ScopedLock lock(mutex_); @@ -1253,6 +1454,117 @@ CompressionError BrotliDecoderContext::GetErrorInfo() const { } } +void ZstdContext::Close() {} + +void ZstdContext::SetBuffers(const char* in, + uint32_t in_len, + char* out, + uint32_t out_len) { + input_.src = reinterpret_cast(in); + input_.size = in_len; + input_.pos = 0; + + output_.dst = reinterpret_cast(out); + output_.size = out_len; + output_.pos = 0; +} + +void ZstdContext::SetFlush(int flush) { + flush_ = static_cast(flush); +} + +void ZstdContext::GetAfterWriteOffsets(uint32_t* avail_in, + uint32_t* avail_out) const { + *avail_in = input_.size - input_.pos; + *avail_out = output_.size - output_.pos; +} + +CompressionError ZstdContext::GetErrorInfo() const { + if (error_ != ZSTD_error_no_error) { + return CompressionError(error_string_.c_str(), + error_code_string_.c_str(), + static_cast(error_)); + } else { + return CompressionError{}; + } +} + +CompressionError ZstdCompressContext::SetParameter(int key, int value) { + size_t result = ZSTD_CCtx_setParameter( + cctx_.get(), static_cast(key), value); + if (ZSTD_isError(result)) { + return CompressionError( + "Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1); + } + return CompressionError{}; +} + +CompressionError ZstdCompressContext::Init(uint64_t pledged_src_size) { + pledged_src_size_ = pledged_src_size; + cctx_.reset(ZSTD_createCCtx()); + if (!cctx_) { + return CompressionError("Could not initialize zstd instance", + "ERR_ZLIB_INITIALIZATION_FAILED", + -1); + } + size_t result = ZSTD_CCtx_setPledgedSrcSize(cctx_.get(), pledged_src_size); + if (ZSTD_isError(result)) { + return CompressionError( + "Could not set pledged src size", "ERR_ZLIB_INITIALIZATION_FAILED", -1); + } + return CompressionError{}; +} + +CompressionError ZstdCompressContext::ResetStream() { + return Init(pledged_src_size_); +} + +void ZstdCompressContext::DoThreadPoolWork() { + size_t const remaining = + ZSTD_compressStream2(cctx_.get(), &output_, &input_, flush_); + if (ZSTD_isError(remaining)) { + error_ = ZSTD_getErrorCode(remaining); + error_code_string_ = ZstdStrerror(error_); + error_string_ = ZSTD_getErrorString(error_); + return; + } +} + +CompressionError ZstdDecompressContext::SetParameter(int key, int value) { + size_t result = ZSTD_DCtx_setParameter( + dctx_.get(), static_cast(key), value); + if (ZSTD_isError(result)) { + return CompressionError( + "Setting parameter failed", "ERR_ZSTD_PARAM_SET_FAILED", -1); + } + return CompressionError{}; +} + +CompressionError ZstdDecompressContext::Init(uint64_t pledged_src_size) { + dctx_.reset(ZSTD_createDCtx()); + if (!dctx_) { + return CompressionError("Could not initialize zstd instance", + "ERR_ZLIB_INITIALIZATION_FAILED", + -1); + } + return CompressionError{}; +} + +CompressionError ZstdDecompressContext::ResetStream() { + // We pass ZSTD_CONTENTSIZE_UNKNOWN because the argument is ignored for + // decompression. + return Init(ZSTD_CONTENTSIZE_UNKNOWN); +} + +void ZstdDecompressContext::DoThreadPoolWork() { + size_t const ret = ZSTD_decompressStream(dctx_.get(), &output_, &input_); + if (ZSTD_isError(ret)) { + error_ = ZSTD_getErrorCode(ret); + error_code_string_ = ZstdStrerror(error_); + error_string_ = ZSTD_getErrorString(error_); + return; + } +} template struct MakeClass { @@ -1322,6 +1634,8 @@ void Initialize(Local target, MakeClass::Make(env, target, "Zlib"); MakeClass::Make(env, target, "BrotliEncoder"); MakeClass::Make(env, target, "BrotliDecoder"); + MakeClass::Make(env, target, "ZstdCompress"); + MakeClass::Make(env, target, "ZstdDecompress"); SetMethod(context, target, "crc32", CRC32); target->Set(env->context(), @@ -1333,6 +1647,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) { MakeClass::Make(registry); MakeClass::Make(registry); MakeClass::Make(registry); + MakeClass::Make(registry); + MakeClass::Make(registry); registry->Register(CRC32); } @@ -1377,6 +1693,8 @@ void DefineZlibConstants(Local target) { NODE_DEFINE_CONSTANT(target, UNZIP); NODE_DEFINE_CONSTANT(target, BROTLI_DECODE); NODE_DEFINE_CONSTANT(target, BROTLI_ENCODE); + NODE_DEFINE_CONSTANT(target, ZSTD_DECOMPRESS); + NODE_DEFINE_CONSTANT(target, ZSTD_COMPRESS); NODE_DEFINE_CONSTANT(target, Z_MIN_WINDOWBITS); NODE_DEFINE_CONSTANT(target, Z_MAX_WINDOWBITS); @@ -1456,6 +1774,70 @@ void DefineZlibConstants(Local target) { NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_RING_BUFFER_2); NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_ALLOC_BLOCK_TYPE_TREES); NODE_DEFINE_CONSTANT(target, BROTLI_DECODER_ERROR_UNREACHABLE); + + // Zstd constants + NODE_DEFINE_CONSTANT(target, ZSTD_e_continue); + NODE_DEFINE_CONSTANT(target, ZSTD_e_flush); + NODE_DEFINE_CONSTANT(target, ZSTD_e_end); + NODE_DEFINE_CONSTANT(target, ZSTD_fast); + NODE_DEFINE_CONSTANT(target, ZSTD_dfast); + NODE_DEFINE_CONSTANT(target, ZSTD_greedy); + NODE_DEFINE_CONSTANT(target, ZSTD_lazy); + NODE_DEFINE_CONSTANT(target, ZSTD_lazy2); + NODE_DEFINE_CONSTANT(target, ZSTD_btlazy2); + NODE_DEFINE_CONSTANT(target, ZSTD_btopt); + NODE_DEFINE_CONSTANT(target, ZSTD_btultra); + NODE_DEFINE_CONSTANT(target, ZSTD_btultra2); + NODE_DEFINE_CONSTANT(target, ZSTD_c_compressionLevel); + NODE_DEFINE_CONSTANT(target, ZSTD_c_windowLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_hashLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_chainLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_searchLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_minMatch); + NODE_DEFINE_CONSTANT(target, ZSTD_c_targetLength); + NODE_DEFINE_CONSTANT(target, ZSTD_c_strategy); + NODE_DEFINE_CONSTANT(target, ZSTD_c_enableLongDistanceMatching); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmMinMatch); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmBucketSizeLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_ldmHashRateLog); + NODE_DEFINE_CONSTANT(target, ZSTD_c_contentSizeFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_checksumFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_dictIDFlag); + NODE_DEFINE_CONSTANT(target, ZSTD_c_nbWorkers); + NODE_DEFINE_CONSTANT(target, ZSTD_c_jobSize); + NODE_DEFINE_CONSTANT(target, ZSTD_c_overlapLog); + NODE_DEFINE_CONSTANT(target, ZSTD_d_windowLogMax); + NODE_DEFINE_CONSTANT(target, ZSTD_CLEVEL_DEFAULT); + // Error codes + NODE_DEFINE_CONSTANT(target, ZSTD_error_no_error); + NODE_DEFINE_CONSTANT(target, ZSTD_error_GENERIC); + NODE_DEFINE_CONSTANT(target, ZSTD_error_prefix_unknown); + NODE_DEFINE_CONSTANT(target, ZSTD_error_version_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_frameParameter_windowTooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_corruption_detected); + NODE_DEFINE_CONSTANT(target, ZSTD_error_checksum_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_literals_headerWrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_corrupted); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionary_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dictionaryCreation_failed); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_combination_unsupported); + NODE_DEFINE_CONSTANT(target, ZSTD_error_parameter_outOfBound); + NODE_DEFINE_CONSTANT(target, ZSTD_error_tableLog_tooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooLarge); + NODE_DEFINE_CONSTANT(target, ZSTD_error_maxSymbolValue_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_stabilityCondition_notRespected); + NODE_DEFINE_CONSTANT(target, ZSTD_error_stage_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_init_missing); + NODE_DEFINE_CONSTANT(target, ZSTD_error_memory_allocation); + NODE_DEFINE_CONSTANT(target, ZSTD_error_workSpace_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dstSize_tooSmall); + NODE_DEFINE_CONSTANT(target, ZSTD_error_srcSize_wrong); + NODE_DEFINE_CONSTANT(target, ZSTD_error_dstBuffer_null); + NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_destFull); + NODE_DEFINE_CONSTANT(target, ZSTD_error_noForwardProgress_inputEmpty); } } // namespace node diff --git a/test/fixtures/person.jpg.zst b/test/fixtures/person.jpg.zst new file mode 100644 index 00000000000000..ea2df5c9a6668a Binary files /dev/null and b/test/fixtures/person.jpg.zst differ diff --git a/test/parallel/test-zlib-bytes-read.js b/test/parallel/test-zlib-bytes-read.js index 605281cd3bd7e6..09b515cc7b70cf 100644 --- a/test/parallel/test-zlib-bytes-read.js +++ b/test/parallel/test-zlib-bytes-read.js @@ -33,6 +33,7 @@ for (const method of [ ['createDeflate', 'createInflate', true], ['createDeflateRaw', 'createInflateRaw', true], ['createBrotliCompress', 'createBrotliDecompress', true], + ['createZstdCompress', 'createZstdDecompress', false], ]) { let compWriter; let compData = Buffer.alloc(0); diff --git a/test/parallel/test-zlib-convenience-methods.js b/test/parallel/test-zlib-convenience-methods.js index 01ec7e211bd5aa..f938a1f6a756ca 100644 --- a/test/parallel/test-zlib-convenience-methods.js +++ b/test/parallel/test-zlib-convenience-methods.js @@ -54,6 +54,8 @@ for (const [type, expect] of [ ['deflateRaw', 'inflateRaw', 'DeflateRaw', 'InflateRaw'], ['brotliCompress', 'brotliDecompress', 'BrotliCompress', 'BrotliDecompress'], + ['zstdCompress', 'zstdDecompress', + 'ZstdCompress', 'ZstdDecompress'], ]) { zlib[method[0]](expect, opts, common.mustCall((err, result) => { zlib[method[1]](result, opts, common.mustCall((err, result) => { diff --git a/test/parallel/test-zlib-empty-buffer.js b/test/parallel/test-zlib-empty-buffer.js index 27fd1340fd1eb4..af53b3013ee82f 100644 --- a/test/parallel/test-zlib-empty-buffer.js +++ b/test/parallel/test-zlib-empty-buffer.js @@ -11,10 +11,12 @@ const emptyBuffer = Buffer.alloc(0); [ zlib.deflateSync, zlib.inflateSync, 'deflate sync' ], [ zlib.gzipSync, zlib.gunzipSync, 'gzip sync' ], [ zlib.brotliCompressSync, zlib.brotliDecompressSync, 'br sync' ], + [ zlib.zstdCompressSync, zlib.zstdDecompressSync, 'zstd sync' ], [ promisify(zlib.deflateRaw), promisify(zlib.inflateRaw), 'raw' ], [ promisify(zlib.deflate), promisify(zlib.inflate), 'deflate' ], [ promisify(zlib.gzip), promisify(zlib.gunzip), 'gzip' ], [ promisify(zlib.brotliCompress), promisify(zlib.brotliDecompress), 'br' ], + [ promisify(zlib.zstdCompress), promisify(zlib.zstdDecompress), 'zstd' ], ]) { const compressed = await compress(emptyBuffer); const decompressed = await decompress(compressed); diff --git a/test/parallel/test-zlib-invalid-input.js b/test/parallel/test-zlib-invalid-input.js index 7aa44dfe7090a1..7aea0efa06dbcf 100644 --- a/test/parallel/test-zlib-invalid-input.js +++ b/test/parallel/test-zlib-invalid-input.js @@ -40,6 +40,7 @@ const unzips = [ zlib.Inflate(), zlib.InflateRaw(), zlib.BrotliDecompress(), + zlib.ZstdDecompress(), ]; nonStringInputs.forEach(common.mustCall((input) => { diff --git a/test/parallel/test-zlib-random-byte-pipes.js b/test/parallel/test-zlib-random-byte-pipes.js index d8d039a6d65c4f..6a7d7c505e5c25 100644 --- a/test/parallel/test-zlib-random-byte-pipes.js +++ b/test/parallel/test-zlib-random-byte-pipes.js @@ -144,6 +144,7 @@ class HashStream extends Stream { for (const [ createCompress, createDecompress ] of [ [ zlib.createGzip, zlib.createGunzip ], [ zlib.createBrotliCompress, zlib.createBrotliDecompress ], + [ zlib.createZstdCompress, zlib.createZstdDecompress ], ]) { const inp = new RandomReadStream({ total: 1024, block: 256, jitter: 16 }); const out = new HashStream(); diff --git a/test/parallel/test-zlib-write-after-flush.js b/test/parallel/test-zlib-write-after-flush.js index c3706678508a93..fbe7c9859ea9fe 100644 --- a/test/parallel/test-zlib-write-after-flush.js +++ b/test/parallel/test-zlib-write-after-flush.js @@ -28,9 +28,10 @@ const zlib = require('node:zlib'); const { test } = require('node:test'); test('zlib should accept writing after flush', async () => { - for (const [createCompress, createDecompress] of [ - [zlib.createGzip, zlib.createGunzip], - [zlib.createBrotliCompress, zlib.createBrotliDecompress], + for (const [ createCompress, createDecompress ] of [ + [ zlib.createGzip, zlib.createGunzip ], + [ zlib.createBrotliCompress, zlib.createBrotliDecompress ], + [ zlib.createZstdCompress, zlib.createZstdDecompress ], ]) { const { promise, resolve, reject } = Promise.withResolvers(); const gzip = createCompress(); diff --git a/test/parallel/test-zlib-zero-byte.js b/test/parallel/test-zlib-zero-byte.js index 7e547b40fadf6c..f75e1d05e9995d 100644 --- a/test/parallel/test-zlib-zero-byte.js +++ b/test/parallel/test-zlib-zero-byte.js @@ -28,7 +28,13 @@ const zlib = require('node:zlib'); const { test } = require('node:test'); test('zlib should properly handle zero byte input', async () => { - for (const Compressor of [zlib.Gzip, zlib.BrotliCompress]) { + const compressors = [ + [zlib.Gzip, 20], + [zlib.BrotliCompress, 1], + [zlib.ZstdCompress, 9], + ]; + + for (const [Compressor, expected] of compressors) { const { promise, resolve, reject } = Promise.withResolvers(); const gz = Compressor(); const emptyBuffer = Buffer.alloc(0); @@ -38,7 +44,6 @@ test('zlib should properly handle zero byte input', async () => { }); gz.on('error', reject); gz.on('end', function() { - const expected = Compressor === zlib.Gzip ? 20 : 1; assert.strictEqual(received, expected, `${received}, ${expected}, ${Compressor.name}`); resolve(); diff --git a/test/parallel/test-zlib-zstd-flush.js b/test/parallel/test-zlib-zstd-flush.js new file mode 100644 index 00000000000000..bde202867bf56d --- /dev/null +++ b/test/parallel/test-zlib-zstd-flush.js @@ -0,0 +1,28 @@ +'use strict'; +require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); +const fixtures = require('../common/fixtures'); + +const file = fixtures.readSync('person.jpg'); +const chunkSize = 16; +const compress = new zlib.ZstdCompress(); + +const chunk = file.slice(0, chunkSize); +const expectedFull = Buffer.from('KLUv/QBYgAAA/9j/4AAQSkZJRgABAQEASA==', 'base64'); +let actualFull; + +compress.write(chunk, function() { + compress.flush(function() { + const bufs = []; + let buf; + while ((buf = compress.read()) !== null) + bufs.push(buf); + actualFull = Buffer.concat(bufs); + }); +}); + +process.once('exit', function() { + assert.deepStrictEqual(actualFull.toString('base64'), expectedFull.toString('base64')); + assert.deepStrictEqual(actualFull, expectedFull); +}); diff --git a/test/parallel/test-zlib-zstd-from-string.js b/test/parallel/test-zlib-zstd-from-string.js new file mode 100644 index 00000000000000..478e3be82604ed --- /dev/null +++ b/test/parallel/test-zlib-zstd-from-string.js @@ -0,0 +1,38 @@ +'use strict'; +// Test compressing and uncompressing a string with zstd + +const common = require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); + +const inputString = 'ΩΩLorem ipsum dolor sit amet, consectetur adipiscing eli' + + 't. Morbi faucibus, purus at gravida dictum, libero arcu ' + + 'convallis lacus, in commodo libero metus eu nisi. Nullam' + + ' commodo, neque nec porta placerat, nisi est fermentum a' + + 'ugue, vitae gravida tellus sapien sit amet tellus. Aenea' + + 'n non diam orci. Proin quis elit turpis. Suspendisse non' + + ' diam ipsum. Suspendisse nec ullamcorper odio. Vestibulu' + + 'm arcu mi, sodales non suscipit id, ultrices ut massa. S' + + 'ed ac sem sit amet arcu malesuada fermentum. Nunc sed. '; +const compressedString = 'KLUv/QRYRQkA9tc9H6AlhTb/z/7/gbTI3kaWLKnbCtkZu/hXm0j' + + 'FpNz/VQM2ADMANQBHTuQOpIYzfVv7XGwXrpoIfgXNAB98xW4wV3' + + 'vnCF2bjcvWZF2wIZ1vr1mSHHvPHU0TgMGBwUFrF0xqReWcWPO8z' + + 'Ny6wMwFUilN+Lg987Zvs2GSRMy6uYvtovK9Uuhgst6l9FQrXLnA' + + '5gpZL7PdI8bO9sDH3tHm73XBzaUK+LjSPNKRmzQ3ZMYEPozdof1' + + '2KcZGfIcLa0PTsdkYqhGcAx/E9mWa8EGEeq0Qou2LTmzgg3YJz/' + + '21OuXSF+TOd662d60Qyb04xC5dOF4b8JFH8mpHAxAAELu3tg1oa' + + 'bBEIWaRHdE0l/+0RdEWWIVMAku8TgbiX/4bU+OpLo4UuY1FKDR8' + + 'RgBc'; + +zlib.zstdCompress(inputString, common.mustCall((err, buffer) => { + assert(inputString.length > buffer.length); + + zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => { + assert.strictEqual(buffer.toString(), inputString); + })); +})); + +const buffer = Buffer.from(compressedString, 'base64'); +zlib.zstdDecompress(buffer, common.mustCall((err, buffer) => { + assert.strictEqual(buffer.toString(), inputString); +})); diff --git a/test/parallel/test-zlib-zstd-from-zstd.js b/test/parallel/test-zlib-zstd-from-zstd.js new file mode 100644 index 00000000000000..628dbefb25cb4e --- /dev/null +++ b/test/parallel/test-zlib-zstd-from-zstd.js @@ -0,0 +1,34 @@ +'use strict'; +// Test unzipping a file that was created with a non-node zstd lib, +// piped in as fast as possible. +// +// The compressed fixture was created using the reference CLI: +// $ zstd -19 test/fixtures/person.jpg -o test/fixtures/person.jpg.zst + +const common = require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); +const fixtures = require('../common/fixtures'); + +const tmpdir = require('../common/tmpdir'); +tmpdir.refresh(); + +const decompress = new zlib.ZstdDecompress(); + +const fs = require('fs'); + +const fixture = fixtures.path('person.jpg.zst'); +const unzippedFixture = fixtures.path('person.jpg'); +const outputFile = tmpdir.resolve('person.jpg'); +const expect = fs.readFileSync(unzippedFixture); +const inp = fs.createReadStream(fixture); +const out = fs.createWriteStream(outputFile); + +inp.pipe(decompress).pipe(out); +out.on('close', common.mustCall(() => { + const actual = fs.readFileSync(outputFile); + assert.strictEqual(actual.length, expect.length); + for (let i = 0, l = actual.length; i < l; i++) { + assert.strictEqual(actual[i], expect[i], `byte[${i}]`); + } +})); diff --git a/test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js b/test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js new file mode 100644 index 00000000000000..58ad8ff2c98dea --- /dev/null +++ b/test/parallel/test-zlib-zstd-kmaxlength-rangeerror.js @@ -0,0 +1,29 @@ +'use strict'; +require('../common'); + +// This test ensures that zlib throws a RangeError if the final buffer needs to +// be larger than kMaxLength and concatenation fails. +// https://github.com/nodejs/node/pull/1811 + +const assert = require('assert'); + +// Change kMaxLength for zlib to trigger the error without having to allocate +// large Buffers. +const buffer = require('buffer'); +const oldkMaxLength = buffer.kMaxLength; +buffer.kMaxLength = 64; +const zlib = require('zlib'); +buffer.kMaxLength = oldkMaxLength; + +// "a".repeat(128), compressed using zstd. +const encoded = Buffer.from('KLUv/SCARQAAEGFhAQA7BVg=', 'base64'); + +// Async +zlib.zstdDecompress(encoded, function(err) { + assert.ok(err instanceof RangeError); +}); + +// Sync +assert.throws(function() { + zlib.zstdDecompressSync(encoded); +}, RangeError); diff --git a/test/parallel/test-zlib-zstd-pledged-src-size.js b/test/parallel/test-zlib-zstd-pledged-src-size.js new file mode 100644 index 00000000000000..b1e32e14ae732a --- /dev/null +++ b/test/parallel/test-zlib-zstd-pledged-src-size.js @@ -0,0 +1,37 @@ +'use strict'; +const common = require('../common'); +const assert = require('assert'); +const zlib = require('zlib'); + +function compressWithPledgedSrcSize({ pledgedSrcSize, actualSrcSize }) { + return new Promise((resolve, reject) => { + const compressor = zlib.createZstdCompress({ pledgedSrcSize }); + compressor.on('error', (e) => { + reject(e); + }); + compressor.on('end', resolve); + compressor.write('x'.repeat(actualSrcSize), () => { + compressor.end(); + compressor.resume(); + }); + }).then(() => { + // Compression should only succeed if sizes match + assert.strictEqual(pledgedSrcSize, actualSrcSize); + }, (error) => { + assert.strictEqual(error.code, 'ZSTD_error_srcSize_wrong'); + // Size error should only happen when sizes do not match + assert.notStrictEqual(pledgedSrcSize, actualSrcSize); + }).then(common.mustCall()); +} + +compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 0 }); + +compressWithPledgedSrcSize({ pledgedSrcSize: 0, actualSrcSize: 42 }); + +compressWithPledgedSrcSize({ pledgedSrcSize: 13, actualSrcSize: 42 }); + +compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 0 }); + +compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 13 }); + +compressWithPledgedSrcSize({ pledgedSrcSize: 42, actualSrcSize: 42 }); diff --git a/test/parallel/test-zlib-zstd.js b/test/parallel/test-zlib-zstd.js new file mode 100644 index 00000000000000..9aad22b206e196 --- /dev/null +++ b/test/parallel/test-zlib-zstd.js @@ -0,0 +1,134 @@ +'use strict'; +require('../common'); +const fixtures = require('../common/fixtures'); +const assert = require('assert'); +const zlib = require('zlib'); + +// Test some zstd-specific properties of the zstd streams that can not +// be easily covered through expanding zlib-only tests. + +const sampleBuffer = fixtures.readSync('/pss-vectors.json'); + +{ + // Test setting the quality parameter at stream creation: + const sizes = []; + for (let quality = 1; + quality <= 22; + quality++) { + const encoded = zlib.zstdCompressSync(sampleBuffer, { + params: { + [zlib.constants.ZSTD_c_compressionLevel]: quality + } + }); + sizes.push(encoded.length); + } + + // Increasing quality should roughly correspond to decreasing compressed size: + for (let i = 0; i < sizes.length - 1; i++) { + assert(sizes[i + 1] <= sizes[i] * 1.05, sizes); // 5 % margin of error. + } + assert(sizes[0] > sizes[sizes.length - 1], sizes); +} + +{ + // Test that setting out-of-bounds option values or keys fails. + assert.throws(() => { + zlib.createZstdCompress({ + params: { + 10000: 0 + } + }); + }, { + code: 'ERR_ZSTD_INVALID_PARAM', + name: 'RangeError', + message: '10000 is not a valid zstd parameter' + }); + + // Test that accidentally using duplicate keys fails. + assert.throws(() => { + zlib.createZstdCompress({ + params: { + '0': 0, + '00': 0 + } + }); + }, { + code: 'ERR_ZSTD_INVALID_PARAM', + name: 'RangeError', + message: '00 is not a valid zstd parameter' + }); + + assert.throws(() => { + zlib.createZstdCompress({ + params: { + // This param must be a valid ZSTD_strategy value. + [zlib.constants.ZSTD_c_strategy]: 130 + } + }); + }, { + code: 'ERR_ZLIB_INITIALIZATION_FAILED', + name: 'Error', + message: 'Initialization failed' + }); + + // Test that setting out-of-bounds option values or keys fails. + assert.throws(() => { + zlib.createZstdDecompress({ + params: { + 10000: 0 + } + }); + }, { + code: 'ERR_ZSTD_INVALID_PARAM', + name: 'RangeError', + message: '10000 is not a valid zstd parameter' + }); + + // Test that accidentally using duplicate keys fails. + assert.throws(() => { + zlib.createZstdDecompress({ + params: { + '0': 0, + '00': 0 + } + }); + }, { + code: 'ERR_ZSTD_INVALID_PARAM', + name: 'RangeError', + message: '00 is not a valid zstd parameter' + }); + + assert.throws(() => { + zlib.createZstdDecompress({ + params: { + // This param must be >= 10 (ZSTD_WINDOWLOG_ABSOLUTEMIN). + [zlib.constants.ZSTD_d_windowLogMax]: 1 + } + }); + }, { + code: 'ERR_ZLIB_INITIALIZATION_FAILED', + name: 'Error', + message: 'Initialization failed' + }); +} + +{ + // Test options.flush range + assert.throws(() => { + zlib.zstdCompressSync('', { flush: zlib.constants.Z_FINISH }); + }, { + code: 'ERR_OUT_OF_RANGE', + name: 'RangeError', + message: 'The value of "options.flush" is out of range. It must be >= 0 ' + + 'and <= 2. Received 4', + }); + + assert.throws(() => { + zlib.zstdCompressSync('', { finishFlush: zlib.constants.Z_FINISH }); + }, { + code: 'ERR_OUT_OF_RANGE', + name: 'RangeError', + message: 'The value of "options.finishFlush" is out of range. It must be ' + + '>= 0 and <= 2. Received 4', + }); +} diff --git a/test/parallel/test-zlib.js b/test/parallel/test-zlib.js index 65050b85a036cf..3b8878d915c097 100644 --- a/test/parallel/test-zlib.js +++ b/test/parallel/test-zlib.js @@ -42,6 +42,7 @@ let zlibPairs = [ [zlib.Gzip, zlib.Unzip], [zlib.DeflateRaw, zlib.InflateRaw], [zlib.BrotliCompress, zlib.BrotliDecompress], + [zlib.ZstdCompress, zlib.ZstdDecompress], ]; // How fast to trickle through the slowstream diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs index 6b94a94283ccb2..be4c17a1f8c4a2 100644 --- a/tools/doc/type-parser.mjs +++ b/tools/doc/type-parser.mjs @@ -245,6 +245,7 @@ const customTypesMap = { 'X509Certificate': 'crypto.html#class-x509certificate', 'zlib options': 'zlib.html#class-options', + 'zstd options': 'zlib.html#class-zstdoptions', 'ReadableStream': 'webstreams.html#class-readablestream',