From 3412296a768d2e587088244d8212fc8ef33fb821 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:37:11 +0200 Subject: [PATCH 1/6] feat: working decompression --- package-lock.json | 6 ++ package.json | 1 + src/core/operations/ZStandardDecompress.mjs | 90 +++++++++++++++++++++ 3 files changed, 97 insertions(+) create mode 100644 src/core/operations/ZStandardDecompress.mjs diff --git a/package-lock.json b/package-lock.json index 3904f84080..70cf322700 100644 --- a/package-lock.json +++ b/package-lock.json @@ -44,6 +44,7 @@ "fernet": "^0.4.0", "file-saver": "^2.0.5", "flat": "^6.0.1", + "fzstd": "^0.1.1", "geodesy": "1.1.3", "highlight.js": "^11.9.0", "ieee754": "^1.2.1", @@ -7916,6 +7917,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/fzstd": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/fzstd/-/fzstd-0.1.1.tgz", + "integrity": "sha512-dkuVSOKKwh3eas5VkJy1AW1vFpet8TA/fGmVA5krThl8YcOVE/8ZIoEA1+U1vEn5ckxxhLirSdY837azmbaNHA==" + }, "node_modules/gamma": { "version": "1.0.0", "license": "MIT" diff --git a/package.json b/package.json index cc3517d356..e092ad497d 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "fernet": "^0.4.0", "file-saver": "^2.0.5", "flat": "^6.0.1", + "fzstd": "^0.1.1", "geodesy": "1.1.3", "highlight.js": "^11.9.0", "ieee754": "^1.2.1", diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs new file mode 100644 index 0000000000..9d68a40131 --- /dev/null +++ b/src/core/operations/ZStandardDecompress.mjs @@ -0,0 +1,90 @@ +/** + * @author Scarjit [ferdinand@linnenberg.dev] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +global.document = {}; +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; +import {isWorkerEnvironment} from "../Utils.mjs"; +import * as fzstd from "fzstd"; + +/** + * ZStandard Decompress operation + */ +class ZStandardDecompress extends Operation { + + /** + * ZStandardDecompress constructor + */ + constructor() { + super(); + + this.name = "ZStandard Decompress"; + this.module = "Compress"; + this.description = "ZStandard is a compression algorithm focused on fast decompression."; + this.infoURL = "https://wikipedia.org/wiki/Zstd"; // Usually a Wikipedia link. Remember to remove localisation (i.e. https://wikipedia.org/etc rather than https://en.wikipedia.org/etc) + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + { + "name": "Chunk Size (bytes)", + "type": "number", + "value": 65536 + } + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + const chunkSize = args[0]; + if (input.byteLength <= 0) { + throw new OperationError("Please provide an input."); + } + // Validate input starts with ZStandard magic number + const magicNumber = new Uint8Array(input, 0, 4); + if (magicNumber[0] !== 0x28 || magicNumber[1] !== 0xb5 || magicNumber[2] !== 0x2f || magicNumber[3] !== 0xfd) { + throw new OperationError("Invalid ZStandard input: does not start with magic number."); + } + + if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); + return new Promise(async (resolve, reject) => { + const compressed = new Uint8Array(input); + try { + const outChunks = []; // Array of Uint8Array chunks + const stream = new fzstd.Decompress((chunk, isLast) => { + // Add to list of output chunks + outChunks.push(chunk); + // Log after all chunks decompressed + if (isLast) { + // Combine all chunks into a single Uint8Array + const totalLength = outChunks.reduce((sum, chunk) => sum + chunk.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of outChunks) { + result.set(chunk, offset); + offset += chunk.length; + } + resolve(result.buffer); + } + }); + for (let i = 0; i < compressed.length; i += chunkSize) { + if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${Math.ceil(compressed.length / chunkSize)}`); + const chunk = compressed.subarray(i, i + chunkSize); + stream.push(chunk); + } + stream.push(new Uint8Array(0), true); // Signal end of stream + } catch (error) { + reject(new OperationError("Decompression failed: " + error.message)); + } + }); + } + +} + +export default ZStandardDecompress; From adca9f3e400ac3f7256c133737f7d6c7248807c0 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:46:37 +0200 Subject: [PATCH 2/6] chore: cleanup --- src/core/operations/ZStandardDecompress.mjs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs index 9d68a40131..a6a5376854 100644 --- a/src/core/operations/ZStandardDecompress.mjs +++ b/src/core/operations/ZStandardDecompress.mjs @@ -4,7 +4,6 @@ * @license Apache-2.0 */ -global.document = {}; import Operation from "../Operation.mjs"; import OperationError from "../errors/OperationError.mjs"; import {isWorkerEnvironment} from "../Utils.mjs"; @@ -47,20 +46,21 @@ class ZStandardDecompress extends Operation { throw new OperationError("Please provide an input."); } // Validate input starts with ZStandard magic number + const ZSTD_MAGIC_NUMBER = [0x28, 0xb5, 0x2f, 0xfd]; const magicNumber = new Uint8Array(input, 0, 4); - if (magicNumber[0] !== 0x28 || magicNumber[1] !== 0xb5 || magicNumber[2] !== 0x2f || magicNumber[3] !== 0xfd) { + if (!ZSTD_MAGIC_NUMBER.every((val, index) => val === magicNumber[index])) { throw new OperationError("Invalid ZStandard input: does not start with magic number."); } + if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); - return new Promise(async (resolve, reject) => { + return new Promise( (resolve, reject) => { const compressed = new Uint8Array(input); try { const outChunks = []; // Array of Uint8Array chunks const stream = new fzstd.Decompress((chunk, isLast) => { - // Add to list of output chunks + // Add to the list of output chunks outChunks.push(chunk); - // Log after all chunks decompressed if (isLast) { // Combine all chunks into a single Uint8Array const totalLength = outChunks.reduce((sum, chunk) => sum + chunk.length, 0); @@ -73,8 +73,9 @@ class ZStandardDecompress extends Operation { resolve(result.buffer); } }); + const chunks = Math.ceil(compressed.length / chunkSize); for (let i = 0; i < compressed.length; i += chunkSize) { - if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${Math.ceil(compressed.length / chunkSize)}`); + if (isWorkerEnvironment()) self.sendStatusMessage(`Decompressing chunk ${i / chunkSize + 1} of ${chunks}...`); const chunk = compressed.subarray(i, i + chunkSize); stream.push(chunk); } From b6944b7ab6c249c2fe760b285e35aded33a62700 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 10:58:58 +0200 Subject: [PATCH 3/6] feat: added test --- src/core/config/Categories.json | 3 ++- tests/operations/tests/Compress.mjs | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/core/config/Categories.json b/src/core/config/Categories.json index bebdd6a5e2..aa3095bb82 100644 --- a/src/core/config/Categories.json +++ b/src/core/config/Categories.json @@ -381,7 +381,8 @@ "LZMA Compress", "LZ4 Decompress", "LZ4 Compress", - "LZNT1 Decompress" + "LZNT1 Decompress", + "ZStandard Decompress" ] }, { diff --git a/tests/operations/tests/Compress.mjs b/tests/operations/tests/Compress.mjs index 60117c675a..88e17db4e9 100644 --- a/tests/operations/tests/Compress.mjs +++ b/tests/operations/tests/Compress.mjs @@ -105,4 +105,19 @@ TestRegister.addTests([ } ], }, + { + name: "ZStandard Decompress", + input: "KLUv/QRYwQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4KpvSd8w==", + expectedOutput: "The cat sat on the mat.", + recipeConfig: [ + { + "op": "From Base64", + "args": [] + }, + { + "op": "ZStandard Decompress", + "args": [] + } + ], + } ]); From e767a89b6113e975f253d6ab30c48e282ddb1ebe Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 11:20:37 +0200 Subject: [PATCH 4/6] fix: lint --- src/core/operations/ZStandardDecompress.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/operations/ZStandardDecompress.mjs b/src/core/operations/ZStandardDecompress.mjs index a6a5376854..3bdcca9bab 100644 --- a/src/core/operations/ZStandardDecompress.mjs +++ b/src/core/operations/ZStandardDecompress.mjs @@ -54,7 +54,7 @@ class ZStandardDecompress extends Operation { if (isWorkerEnvironment()) self.sendStatusMessage("Loading ZStandard..."); - return new Promise( (resolve, reject) => { + return new Promise((resolve, reject) => { const compressed = new Uint8Array(input); try { const outChunks = []; // Array of Uint8Array chunks From 158641c92128200089a20bca5d6321e388ca2f94 Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Tue, 17 Sep 2024 11:31:15 +0200 Subject: [PATCH 5/6] fix: removed newline from input and set default arg --- tests/operations/tests/Compress.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/operations/tests/Compress.mjs b/tests/operations/tests/Compress.mjs index 88e17db4e9..d1d451fdc2 100644 --- a/tests/operations/tests/Compress.mjs +++ b/tests/operations/tests/Compress.mjs @@ -107,7 +107,7 @@ TestRegister.addTests([ }, { name: "ZStandard Decompress", - input: "KLUv/QRYwQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4KpvSd8w==", + input: "KLUv/QRYuQAAVGhlIGNhdCBzYXQgb24gdGhlIG1hdC4tJ481", expectedOutput: "The cat sat on the mat.", recipeConfig: [ { @@ -116,7 +116,7 @@ TestRegister.addTests([ }, { "op": "ZStandard Decompress", - "args": [] + "args": [65536] } ], } From 0629ad9ab6b6f3d2f9ba0b1fe17f822a4019c2bc Mon Sep 17 00:00:00 2001 From: Ferdinand Linnenberg Date: Fri, 11 Oct 2024 19:41:23 +0200 Subject: [PATCH 6/6] feat: makefile --- Makefile | 43 +++++++++++++++++++ src/core/operations/ZStandardDecode.mjs | 56 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 Makefile create mode 100644 src/core/operations/ZStandardDecode.mjs diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..efe766a3dd --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +# Variables +UPSTREAM_REPO := upstream +UPSTREAM_URL := https://github.com/gchq/CyberChef.git +FORK_REPO := origin +DEFAULT_BRANCH := main + +# Default target if no target is specified +.DEFAULT_GOAL := help + +# Help command +help: + @echo "Usage:" + @echo " make pr PR_ID= - Fetch and push a PR from the upstream repo" + @echo " make setup-upstream - Set up upstream repo (idempotent)" + @echo " make clean PR_ID= - Clean up the local PR branch" + +# Setup upstream repo (idempotent) +setup-upstream: + @echo "Checking if upstream repo exists..." + @if ! git remote get-url $(UPSTREAM_REPO) >/dev/null 2>&1; then \ + echo "Upstream repo not found. Adding upstream..."; \ + git remote add $(UPSTREAM_REPO) $(UPSTREAM_URL); \ + else \ + echo "Upstream repo already exists."; \ + fi + +# Fetch the PR from upstream, create a branch, and push it to your fork +pr: setup-upstream + @if [ -z "$(PR_ID)" ]; then \ + echo "Error: PR_ID is not set. Usage: make pr PR_ID="; \ + exit 1; \ + fi + git fetch $(UPSTREAM_REPO) pull/$(PR_ID)/head:pr-$(PR_ID) + git checkout pr-$(PR_ID) + git push $(FORK_REPO) pr-$(PR_ID) + +# Clean up the local PR branch +clean: + @if [ -z "$(PR_ID)" ]; then \ + echo "Error: PR_ID is not set. Usage: make clean PR_ID="; \ + exit 1; \ + fi + git branch -d pr-$(PR_ID) diff --git a/src/core/operations/ZStandardDecode.mjs b/src/core/operations/ZStandardDecode.mjs new file mode 100644 index 0000000000..b984b19288 --- /dev/null +++ b/src/core/operations/ZStandardDecode.mjs @@ -0,0 +1,56 @@ +/** + * @author Scarjit [ferdinand@linnenberg.dev] + * @copyright Crown Copyright 2024 + * @license Apache-2.0 + */ + +import Operation from "../Operation.mjs"; +import OperationError from "../errors/OperationError.mjs"; + +/** + * ZStandard Decode operation + */ +class ZStandardDecode extends Operation { + + /** + * ZStandardDecode constructor + */ + constructor() { + super(); + + this.name = "ZStandard Decode"; + this.module = "Compression"; + this.description = "Zstandard is a lossless data compression algorithm designed for fast compression and decompression. It was developed by Facebook."; + this.infoURL = "https://wikipedia.org/wiki/Zstd"; // Usually a Wikipedia link. Remember to remove localisation (i.e. https://wikipedia.org/etc rather than https://en.wikipedia.org/etc) + this.inputType = "ArrayBuffer"; + this.outputType = "ArrayBuffer"; + this.args = [ + /* Example arguments. See the project wiki for full details. + { + name: "First arg", + type: "string", + value: "Don't Panic" + }, + { + name: "Second arg", + type: "number", + value: 42 + } + */ + ]; + } + + /** + * @param {ArrayBuffer} input + * @param {Object[]} args + * @returns {ArrayBuffer} + */ + run(input, args) { + // const [firstArg, secondArg] = args; + + throw new OperationError("Test"); + } + +} + +export default ZStandardDecode;