Magic Overhaul

src/core/Utils.mjs

@@ -1182,6 +1182,7 @@ class Utils {
             "CRLF":          /\r\n/g,
             "Forward slash": /\//g,
             "Backslash":     /\\/g,
+            "0x with comma": /,?0x/g,
             "0x":            /0x/g,
             "\\x":           /\\x/g,
             "None":          /\s+/g // Included here to remove whitespace when there shouldn't be any

src/core/config/scripts/generateConfig.mjs

@@ -42,13 +42,32 @@ for (const opObj in Ops) {
         outputType:  op.presentType,
         flowControl: op.flowControl,
         manualBake:  op.manualBake,
-        args:        op.args
+        args:        op.args,
     };
 
-    if ("patterns" in op) {
-        operationConfig[op.name].patterns = op.patterns;
+    if ("checks" in op) {
+        if ("input" in op.checks) {
+            operationConfig[op.name].input = {};
+            if ("regex" in op.checks.input) {
+                operationConfig[op.name].input.regex = op.checks.input.regex;
+            }
+            if ("entropy" in op.checks.input) {
+                operationConfig[op.name].input.entropy = op.checks.input.entropy;
+            }
+        }
+        if ("output" in op.checks) {
+            operationConfig[op.name].output = {};
+            if ("regex" in op.checks.output) {
+                operationConfig[op.name].output.regex = op.checks.output.regex;
+            }
+            if ("entropy" in op.checks.output) {
+                operationConfig[op.name].output.entropy = op.checks.output.entropy;
+            }
+            if ("mime" in op.checks.output) {
+                operationConfig[op.name].output.mime = op.checks.output.mime;
+            }
+        }
     }
-
     if (!(op.module in modules))
         modules[op.module] = {};
     modules[op.module][op.name] = opObj;

src/core/lib/IP.mjs

@@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
     let output = "";
 
     if (cidrRange < 0 || cidrRange > 31) {
-        return "IPv4 CIDR must be less than 32";
+        throw new OperationError("IPv4 CIDR must be less than 32");
     }
 
     const mask = ~(0xFFFFFFFF >>> cidrRange),
@@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
         cidrRange = parseInt(cidr[cidr.length-1], 10);
 
     if (cidrRange < 0 || cidrRange > 127) {
-        return "IPv6 CIDR must be less than 128";
+        throw new OperationError("IPv6 CIDR must be less than 128");
     }
 
     const ip1 = new Array(8),
@@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
         const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
         const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
         if (cidrRange < 0 || cidrRange > 31) {
-            return "IPv4 CIDR must be less than 32";
+            throw new OperationError("IPv4 CIDR must be less than 32");
         }
         const mask = ~(0xFFFFFFFF >>> cidrRange),
             cidrIp1 = network & mask,
@@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
         const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);
 
         if (cidrRange < 0 || cidrRange > 127) {
-            return "IPv6 CIDR must be less than 128";
+            throw new OperationError("IPv6 CIDR must be less than 128");
         }
 
         const cidrIp1 = new Array(8),

src/core/lib/Magic.mjs

@@ -2,7 +2,7 @@ import OperationConfig from "../config/OperationConfig.json";
 import Utils, { isWorkerEnvironment } from "../Utils.mjs";
 import Recipe from "../Recipe.mjs";
 import Dish from "../Dish.mjs";
-import {detectFileType} from "./FileType.mjs";
+import {detectFileType, isType} from "./FileType.mjs";
 import chiSquared from "chi-squared";
 
 /**
@@ -19,35 +19,68 @@ class Magic {
      * Magic constructor.
      *
      * @param {ArrayBuffer} buf
-     * @param {Object[]} [opPatterns]
+     * @param {Object} prevOp
      */
-    constructor(buf, opPatterns) {
+    constructor(buf, opPatterns, prevOp) {
         this.inputBuffer = new Uint8Array(buf);
         this.inputStr = Utils.arrayBufferToStr(buf);
-        this.opPatterns = opPatterns || Magic._generateOpPatterns();
+        this.opPatterns = opPatterns || Magic._generateOpCriteria();
+        this.prevOp = prevOp;
     }
 
     /**
-     * Finds operations that claim to be able to decode the input based on regular
-     * expression matches.
+     * Finds operations that claim to be able to decode the input based on
+     * regular expression matches.
      *
-     * @returns {Object[]}
+     * @param {[Object]} opPatterns
+     * @returns {Array}
      */
-    findMatchingOps() {
+    inputRegexMatch(opPatterns) {
         const matches = [];
 
-        for (let i = 0; i < this.opPatterns.length; i++) {
-            const pattern = this.opPatterns[i],
-                regex = new RegExp(pattern.match, pattern.flags);
+        for (let i = 0; i < opPatterns.length; i++) {
+            const pattern = opPatterns[i];
+
 
-            if (regex.test(this.inputStr)) {
+            if (pattern.match.test(this.inputStr)) {
                 matches.push(pattern);
             }
         }
 
         return matches;
     }
 
+    /**
+     * Finds operations that claim to be able to decode the input based on entropy
+     * matches.
+     *
+     * @param {[Object]} opPatterns
+     * @returns {Array}
+     */
+    entropyInputMatch(opPatterns) {
+        const matches = [];
+
+        const entropyOfInput = this.calcEntropy();
+
+        for (let i = 0; i < opPatterns.length; i++) {
+            const currOp = opPatterns[i];
+            if ((entropyOfInput > currOp.entropy[0]) && (entropyOfInput < currOp.entropy[1]))
+                matches.push(currOp);
+        }
+        return matches;
+    }
+
+    /**
+     * Finds operations that claim to be able to decode the input based on criteria.
+     *
+     * @returns {Object[]}
+     */
+    findMatchingInputOps() {
+        let matches = this.inputRegexMatch(this.opPatterns.regex);
+        matches = matches.concat(this.entropyInputMatch(this.opPatterns.entropy));
+        return [...new Set(matches)];
+    }
+
     /**
      * Attempts to detect the language of the input by comparing its byte frequency
      * to that of several known languages.
@@ -264,6 +297,35 @@ class Magic {
         return results;
     }
 
+    /**
+     *
+     */
+    checkRegexes(regexes) {
+        for (const elem of regexes) {
+            const regex = new RegExp(elem.match, elem.flags);
+            if  (regex.test(this.inputStr))
+                return true;
+        }
+        return false;
+    }
+    /**
+     *
+     */
+    checkOutputFromPrevious() {
+        let score = 0;
+        if ("regex" in this.prevOp.output) {
+            if (this.checkRegexes(this.prevOp.output.regex)) score++;
+        }
+        if ("entropy" in this.prevOp.output) {
+            const inputEntropy = this.calcEntropy();
+            if ((inputEntropy > this.prevOp.output.entropy[0]) && (inputEntropy < this.prevOp.output.entropy[1])) score++;
+        }
+        if ("mime" in this.prevOp.output) {
+            if (isType(this.prevOp.output.mime, this.inputBuffer)) score++;
+        }
+        return score > 0;
+    }
+
     /**
      * Speculatively executes matching operations, recording metadata of each result.
      *
@@ -281,8 +343,15 @@ class Magic {
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
-        const matchingOps = this.findMatchingOps();
 
+        if (this.prevOp) {
+            if ("output" in this.prevOp) {
+                if (!(this.checkOutputFromPrevious())) {
+                    return [];
+                }
+            }
+        }
+        const matchingOps = this.findMatchingInputOps();
         let results = [];
 
         // Record the properties of the current data
@@ -305,8 +374,7 @@ class Magic {
             const opConfig = {
                     op: op.op,
                     args: op.args
-                },
-                output = await this._runRecipe([opConfig]);
+                }, output = await this._runRecipe([opConfig]);
 
             // If the recipe is repeating and returning the same data, do not continue
             if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
@@ -318,7 +386,8 @@ class Magic {
                 return;
             }
 
-            const magic = new Magic(output, this.opPatterns),
+
+            const magic = new Magic(output, this.opPatterns, OperationConfig[op.op]),
                 speculativeResults = await magic.speculativeExecution(
                     depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
 
@@ -330,7 +399,7 @@ class Magic {
             const bfEncodings = await this.bruteForce();
 
             await Promise.all(bfEncodings.map(async enc => {
-                const magic = new Magic(enc.data, this.opPatterns),
+                const magic = new Magic(enc.data, this.opPatterns, undefined),
                     bfResults = await magic.speculativeExecution(
                         depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
 
@@ -447,24 +516,34 @@ class Magic {
      * @private
      * @returns {Object[]}
      */
-    static _generateOpPatterns() {
-        const opPatterns = [];
+    static _generateOpCriteria() {
+        const opCriteria = {
+            regex: [],
+            entropy: []
+        };
 
         for (const op in OperationConfig) {
-            if (!("patterns" in OperationConfig[op])) continue;
-
-            OperationConfig[op].patterns.forEach(pattern => {
-                opPatterns.push({
-                    op: op,
-                    match: pattern.match,
-                    flags: pattern.flags,
-                    args: pattern.args,
-                    useful: pattern.useful || false
-                });
-            });
+            if ("input" in OperationConfig[op]) {
+                if ("regex" in OperationConfig[op].input)
+                    OperationConfig[op].input.regex.forEach(pattern => {
+                        opCriteria.regex.push({
+                            op: op,
+                            match: new RegExp(pattern.match, pattern.flags),
+                            args: pattern.args,
+                            useful: pattern.useful || false
+                        });
+                    });
+                if ("entropy" in OperationConfig[op].input) {
+                    opCriteria.entropy.push({
+                        op: op,
+                        entropy: OperationConfig[op].input.entropy.input,
+                        args:  OperationConfig[op].input.entropy.args
+                    });
+                }
+            }
         }
 
-        return opPatterns;
+        return opCriteria;
     }
 
     /**

src/core/lib/MagicCriteria.mjs

@@ -0,0 +1,12 @@
+/**
+ * Constants for the entropy of text.
+ *
+ * @author n1073645 [n1073645@gmail.com]
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+*/
+export const compressedToDecompressed = [6.5, 8];
+
+export const binary = [1, 1.5];
+
+export const entropyOfText = [3.5, 6];

src/core/operations/A1Z26CipherDecode.mjs

@@ -33,6 +33,42 @@ class A1Z26CipherDecode extends Operation {
                 value: DELIM_OPTIONS
             }
         ];
+        this.checks = {
+            input: {
+                regex: [
+                    {
+                        match:  "^\\s*([12]?[0-9] )+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["Space"]
+                    },
+                    {
+                        match:  "^\\s*([12]?[0-9],)+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["Comma"]
+                    },
+                    {
+                        match:  "^\\s*([12]?[0-9];)+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["Semi-colon"]
+                    },
+                    {
+                        match:  "^\\s*([12]?[0-9]:)+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["Colon"]
+                    },
+                    {
+                        match:  "^\\s*([12]?[0-9]\\n)+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["Line feed"]
+                    },
+                    {
+                        match:  "^\\s*([12]?[0-9]\\r\\n)+[12]?[0-9]\\s*$",
+                        flags:  "",
+                        args:   ["CRLF"]
+                    }
+                ]
+            }
+        };
     }
 
     /**