feat: implement pipelines with built-in deobfuscate (#398)

Apply suggestion from @Copilot

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

chore: add changeset

Author: fraxken

.changeset/grumpy-insects-love.md

@@ -0,0 +1,5 @@
+---
+"@nodesecure/js-x-ray": minor
+---
+
+Implement new pipeline mechanism with a built-in deobfuscate

workspaces/js-x-ray/docs/AstAnalyser.md

@@ -32,6 +32,7 @@ interface AstAnalyserOptions {
    * @default false
    */
   optionalWarnings?: boolean | Iterable<OptionalWarningName>;
+  pipelines?: Pipeline[];
 }
 ```
 

workspaces/js-x-ray/src/AstAnalyser.ts

@@ -17,11 +17,17 @@ import {
   SourceFile,
   type SourceFlags
 } from "./SourceFile.js";
-import { isOneLineExpressionExport } from "./utils/index.js";
 import { JsSourceParser, type SourceParser } from "./JsSourceParser.js";
 import { ProbeRunner, type Probe } from "./ProbeRunner.js";
 import { walkEnter } from "./walker/index.js";
 import * as trojan from "./obfuscators/trojan-source.js";
+import {
+  isOneLineExpressionExport
+} from "./utils/index.js";
+import {
+  PipelineRunner,
+  type Pipeline
+} from "./pipelines/index.js";
 
 export interface Dependency {
   unsafe: boolean;
@@ -85,23 +91,27 @@ export interface AstAnalyserOptions {
    * @default false
    */
   optionalWarnings?: boolean | Iterable<OptionalWarningName>;
+  pipelines?: Pipeline[];
 }
 
 export interface PrepareSourceOptions {
   removeHTMLComments?: boolean;
 }
 
 export class AstAnalyser {
+  #pipelineRunner: PipelineRunner;
   parser: SourceParser;
   probes: Probe[];
 
   constructor(options: AstAnalyserOptions = {}) {
     const {
       customProbes = [],
       optionalWarnings = false,
-      skipDefaultProbes = false
+      skipDefaultProbes = false,
+      pipelines = []
     } = options;
 
+    this.#pipelineRunner = new PipelineRunner(pipelines);
     this.parser = options.customParser ?? new JsSourceParser();
 
     let probes = ProbeRunner.Defaults;
@@ -144,15 +154,15 @@ export class AstAnalyser {
     const body = this.parser.parse(this.prepareSource(str, { removeHTMLComments }), {
       isEcmaScriptModule: Boolean(module)
     });
+
     const source = new SourceFile();
     if (trojan.verify(str)) {
       source.warnings.push(
         generateWarning("obfuscated-code", { value: "trojan-source" })
       );
     }
 
-    const runner = new ProbeRunner(source, this.probes);
-
+    const probeRunner = new ProbeRunner(source, this.probes);
     if (initialize) {
       if (typeof initialize !== "function") {
         throw new TypeError("options.initialize must be a function");
@@ -161,14 +171,15 @@ export class AstAnalyser {
     }
 
     // we walk each AST Nodes, this is a purely synchronous I/O
-    walkEnter(body, function walk(node) {
+    const reducedBody = this.#pipelineRunner.reduce(body);
+    walkEnter(reducedBody, function walk(node) {
       // Skip the root of the AST.
       if (Array.isArray(node)) {
         return;
       }
 
       source.walk(node);
-      const action = runner.walk(node);
+      const action = probeRunner.walk(node);
       if (action === "skip") {
         this.skip();
       }
@@ -180,7 +191,7 @@ export class AstAnalyser {
       }
       finalize(source);
     }
-    runner.finalize();
+    probeRunner.finalize();
 
     // Add oneline-require flag if this is a one-line require expression
     if (isOneLineExpressionExport(body)) {

workspaces/js-x-ray/src/index.ts

@@ -3,3 +3,7 @@ export * from "./JsSourceParser.js";
 export * from "./AstAnalyser.js";
 export * from "./EntryFilesAnalyser.js";
 export * from "./SourceFile.js";
+export {
+  Pipelines,
+  type Pipeline
+} from "./pipelines/index.js";

workspaces/js-x-ray/src/pipelines/Runner.class.ts

@@ -0,0 +1,44 @@
+// Import Third-party Dependencies
+import type { ESTree } from "meriyah";
+
+export interface Pipeline {
+  name: string;
+
+  walk(
+    body: ESTree.Program["body"]
+  ): ESTree.Program["body"];
+}
+
+export class PipelineRunner {
+  #pipelines: Pipeline[];
+
+  constructor(
+    pipelines: Pipeline[]
+  ) {
+    this.#pipelines = removeDuplicatedPipelines(pipelines);
+  }
+
+  reduce(
+    initialBody: ESTree.Program["body"]
+  ): ESTree.Program["body"] {
+    return this.#pipelines.reduce(
+      (body, pipeline) => pipeline.walk(body),
+      initialBody
+    );
+  }
+}
+
+function removeDuplicatedPipelines(
+  pipelines: Pipeline[]
+): Pipeline[] {
+  const seen = new Set<string>();
+
+  return pipelines.filter((pipeline) => {
+    if (seen.has(pipeline.name)) {
+      return false;
+    }
+    seen.add(pipeline.name);
+
+    return true;
+  });
+}

workspaces/js-x-ray/src/pipelines/deobfuscate.ts

@@ -0,0 +1,46 @@
+// Import Third-party Dependencies
+import type { ESTree } from "meriyah";
+import { match } from "ts-pattern";
+import { joinArrayExpression } from "@nodesecure/estree-ast-utils";
+
+// Import Internal Dependencies
+import { walkEnter } from "../walker/index.js";
+import type { Pipeline } from "./Runner.class.js";
+
+export class Deobfuscate implements Pipeline {
+  name = "deobfuscate";
+
+  #withCallExpression(
+    node: ESTree.CallExpression
+  ): ESTree.Node | void {
+    const value = joinArrayExpression(node);
+    if (value !== null) {
+      return {
+        type: "Literal",
+        value,
+        raw: value
+      };
+    }
+
+    return void 0;
+  }
+
+  walk(
+    body: ESTree.Program["body"]
+  ): ESTree.Program["body"] {
+    const self = this;
+    walkEnter(body, function walk(node): void {
+      if (Array.isArray(node)) {
+        return;
+      }
+
+      match(node)
+        .with({ type: "CallExpression" }, (node) => {
+          this.replaceAndSkip(self.#withCallExpression(node));
+        })
+        .otherwise(() => void 0);
+    });
+
+    return body;
+  }
+}

workspaces/js-x-ray/src/pipelines/index.ts

@@ -0,0 +1,13 @@
+// Import Internal Dependencies
+import { Deobfuscate } from "./deobfuscate.js";
+import {
+  PipelineRunner,
+  type Pipeline
+} from "./Runner.class.js";
+
+export const Pipelines = Object.freeze({
+  deobfuscate: Deobfuscate
+}) satisfies Record<string, new() => Pipeline>;
+
+export { PipelineRunner };
+export type { Pipeline };

workspaces/js-x-ray/src/walker/walker.base.ts

@@ -4,7 +4,8 @@ import type { ESTree } from "meriyah";
 export interface WalkerContext {
   skip: () => void;
   remove: () => void;
-  replace: (node: ESTree.Node) => void;
+  replace: (node: ESTree.Node | void) => void;
+  replaceAndSkip: (node: ESTree.Node | void) => void;
 }
 
 export class WalkerBase {
@@ -17,7 +18,17 @@ export class WalkerBase {
     this.context = {
       skip: () => (this.should_skip = true),
       remove: () => (this.should_remove = true),
-      replace: (node) => (this.replacement = node)
+      replace: (node) => {
+        if (node !== undefined) {
+          this.replacement = node;
+        }
+      },
+      replaceAndSkip: (node) => {
+        this.should_skip = true;
+        if (node !== undefined) {
+          this.replacement = node;
+        }
+      }
     };
   }
 

workspaces/js-x-ray/test/Pipelines.spec.ts

@@ -0,0 +1,87 @@
+// Import Node.js Dependencies
+import { describe, mock, test } from "node:test";
+import assert from "node:assert";
+
+// Import Internal Dependencies
+import {
+  AstAnalyser,
+  JsSourceParser,
+  Pipelines
+} from "../src/index.js";
+import {
+  getWarningKind
+} from "./utils/index.js";
+
+describe("AstAnalyser pipelines", () => {
+  test("should iterate once on the pipeline", () => {
+    const pipeline = {
+      name: "test-pipeline",
+      walk: mock.fn((body) => body)
+    };
+
+    const analyser = new AstAnalyser({
+      customParser: new JsSourceParser(),
+      pipelines: [
+        pipeline,
+        pipeline
+      ]
+    });
+
+    analyser.analyse(`return "Hello World";`);
+
+    assert.strictEqual(pipeline.walk.mock.callCount(), 1);
+    assert.deepEqual(
+      pipeline.walk.mock.calls[0].arguments[0],
+      [
+        {
+          type: "ReturnStatement",
+          argument: {
+            type: "Literal",
+            value: "Hello World",
+            raw: "\"Hello World\"",
+            loc: {
+              start: {
+                line: 1,
+                column: 7
+              },
+              end: {
+                line: 1,
+                column: 20
+              }
+            }
+          },
+          loc: {
+            start: {
+              line: 1,
+              column: 0
+            },
+            end: {
+              line: 1,
+              column: 21
+            }
+          }
+        }
+      ]
+    );
+  });
+});
+
+describe("Pipelines.deobfuscate", () => {
+  test("should find a shady-url by deobfuscating a joined ArrayExpression", () => {
+    const analyser = new AstAnalyser({
+      customParser: new JsSourceParser(),
+      pipelines: [
+        new Pipelines.deobfuscate()
+      ]
+    });
+
+    const { warnings } = analyser.analyse(`
+      const URL = ["http://", ["77", "244", "210", "1"].join("."), "/script"].join("");
+    `);
+
+    assert.deepEqual(
+      getWarningKind(warnings),
+      ["shady-link"].sort()
+    );
+  });
+});