ci(custom-checks): fix depcheck not detecting missing dependencies

1. The depcheck tool we use have not correctly discovered some of the
missing dependencies that we have because it only verifies that the
imported dependency is present SOMEwhere in the package.json file, not that
it is specifically present in the production dependencies section which
leads to crashes and broken packages due to the API server not installing
dev dependencies when instantiating a plugin and therefore missing a few
of the dependencies that are otherwise very much needed at runtime in
production.
2. The solution to the problem was to implement our own typescript parsing
with babel and then double check the work of depcheck to make sure that
the dependencies that it marks as "no issues" are actually OK and have no
issues.
3. The hardest edge case was type imports e.g. `import type { Express } from "express";`
because the import was there, but we did not actually need that dependency
in the production dependencies as long as ALL of the imports to it in the
given package were type imports. To robustly verify this being the case or not
we had to pull out the big guns and parse all the typescript code per package
to make sure that we've looked at every single import of the dependency in
question at every single code file of the package in question.

Depends on #3345

Signed-off-by: Peter Somogyvari <peter.somogyvari@accenture.com>

package.json

@@ -109,6 +109,8 @@
     "zod": ">=3.22.3"
   },
   "devDependencies": {
+    "@babel/parser": "7.24.7",
+    "@babel/types": "7.24.7",
     "@bufbuild/buf": "1.30.0",
     "@bufbuild/protobuf": "1.8.0",
     "@bufbuild/protoc-gen-es": "1.8.0",

tools/custom-checks/check-missing-node-deps.ts

@@ -4,6 +4,8 @@ import { RuntimeError } from "run-time-error";
 import fs from "fs-extra";
 import depcheck from "depcheck";
 import { getAllPkgDirs } from "./get-all-pkg-dirs";
+import { ParseResult, parse } from "@babel/parser";
+import { File as BabelFile, ImportDeclaration } from "@babel/types";
 
 export interface ICheckMissingNodeDepsRequest {
   readonly pkgDirsToCheck: Readonly<Array<string>>;
@@ -28,12 +30,14 @@ export async function checkMissingNodeDeps(
     throw new RuntimeError(`req.pkgDirsToCheck parameter cannot be falsy.`);
   }
 
-  if (req.verbose) {
+  const verbose = req.verbose === true;
+  if (verbose) {
     console.log(`${TAG} SCRIPT_DIR=${SCRIPT_DIR}`);
     console.log(`${TAG} PROJECT_DIR=${PROJECT_DIR}`);
+    console.log("%s Package directories to check: %o", TAG, req.pkgDirsToCheck);
   }
 
-  const depCheckOptions = {
+  const depCheckOptions: depcheck.Options = {
     ignoreBinPackage: false, // ignore the packages with bin entry
     skipMissing: false, // skip calculation of missing dependencies
     ignorePatterns: [
@@ -42,6 +46,16 @@ export async function checkMissingNodeDeps(
       "dist",
       "bower_components",
       "node_modules",
+      "docs/",
+      "src/main/kotlin",
+      "src/test/",
+      "Dockerfile",
+      "*.md",
+      "*.json",
+      "*.test.ts",
+      "*.tsx",
+      "*.mts",
+      "*.scss",
     ],
     ignoreMatches: [
       // ignore dependencies that matches these globs
@@ -52,46 +66,54 @@ export async function checkMissingNodeDeps(
       "tap",
       "@ionic-native/*",
     ],
-    // parsers: {
-    //   // the target parsers
-    //   '**/*.js': depcheck.parser.es6,
-    //   '**/*.jsx': depcheck.parser.jsx,
-    // },
     detectors: [
-      // the target detectors
       depcheck.detector.requireCallExpression,
       depcheck.detector.importDeclaration,
       depcheck.detector.importCallExpression,
     ],
-    specials: [
-      // the target special parsers
-      //   depcheck.special.eslint,
-      //   depcheck.special.webpack,
-    ],
   };
 
   const tasks = req.pkgDirsToCheck.map(async (pkgDir) => {
-    const manifest = path.join(pkgDir, "package.json");
-    const manifestExists = await fs.pathExists(manifest);
+    const manifestPath = path.join(pkgDir, "package.json");
+    const manifestExists = await fs.pathExists(manifestPath);
     if (!manifestExists) {
-      if (req.verbose) {
-        console.log("%s %s has no package.json. Skipping.", TAG, manifest);
+      if (verbose) {
+        console.log("%s %s has no package.json. Skipping.", TAG, manifestPath);
       }
       return;
     }
-    const { missing } = await depcheck(pkgDir, depCheckOptions);
+
+    const depCheckResult = await depcheck(pkgDir, depCheckOptions);
+    const { missing } = depCheckResult;
     const missingDepNames = Object.keys(missing);
 
+    if (verbose) {
+      console.log("%s DepCheck result %s: %o", TAG, pkgDir, depCheckResult);
+    }
+
+    const prodCodePathPrefixes = [path.join(pkgDir, "src/main/")];
+
+    const prodDepMissingErrors = await findMissingPrdDeps({
+      depCheckResult,
+      manifestPath,
+      verbose,
+      TAG,
+      prodCodePathPrefixes,
+      ignoredPkgNameRegExps: [new RegExp("@types/.*")],
+    });
+
+    prodDepMissingErrors.forEach((e) => errors.push(e));
+
     missingDepNames.forEach((depName) => {
       const filesUsingDep = missing[depName].join("\n\t");
-      const anError = `"${depName}" is a missing dependency from ${manifest} with the following files using them:\n\t${filesUsingDep}`;
+      const anError = `"${depName}" is a missing dependency from ${manifestPath} with the following files using them:\n\t${filesUsingDep}`;
       errors.push(anError);
       if (req.verbose) {
         console.log(
           "%s Missing dep %s in %s",
           TAG,
           depName,
-          manifest,
+          manifestPath,
           missing[depName],
         );
       }
@@ -103,6 +125,186 @@ export async function checkMissingNodeDeps(
   return [errors.length === 0, errors];
 }
 
+async function findMissingPrdDeps(req: {
+  readonly depCheckResult: depcheck.Results;
+  readonly manifestPath: Readonly<string>;
+  readonly TAG: Readonly<string>;
+  readonly verbose: boolean;
+  readonly prodCodePathPrefixes: Readonly<Array<string>>;
+  readonly ignoredPkgNameRegExps: Readonly<Array<RegExp>>;
+}): Promise<string[]> {
+  const fn =
+    "[tools/custom-checks/check-missing-node-deps.ts]#filterUsingToSrcMainDirs()";
+  if (!req) {
+    throw new Error(fn + "Expected arg req to be truthy");
+  }
+  if (!req.depCheckResult) {
+    throw new Error(fn + "Expected arg req.depCheckResult to be truthy");
+  }
+  if (!req.depCheckResult.using) {
+    throw new Error(fn + "Expected req.depCheckResult.using to be truthy");
+  }
+  if (!req.manifestPath) {
+    throw new Error(fn + "Expected arg req.manifestPath to be truthy");
+  }
+  if (req.verbose !== true && req.verbose !== false) {
+    throw new Error(fn + "Expected arg req.verbose to be strictly bool");
+  }
+  if (!req.TAG) {
+    throw new Error(fn + "Expected arg req.TAG to be truthy");
+  }
+  if (!req.prodCodePathPrefixes) {
+    throw new Error(fn + "Expected arg req.prodCodePathPrefixes to be truthy");
+  }
+  if (!Array.isArray(req.prodCodePathPrefixes)) {
+    throw new Error(fn + "Expected arg req.prodCodePathPrefixes to be Array");
+  }
+  if (!req.ignoredPkgNameRegExps) {
+    throw new Error(fn + "Expected arg req.ignoredPkgNameRegExps to be truthy");
+  }
+  if (!Array.isArray(req.ignoredPkgNameRegExps)) {
+    throw new Error(fn + "Expected arg req.ignoredPkgNameRegExps to be Array");
+  }
+
+  const {
+    manifestPath,
+    depCheckResult,
+    TAG,
+    verbose,
+    prodCodePathPrefixes,
+    ignoredPkgNameRegExps,
+  } = req;
+
+  const prodDepMissingErrors: string[] = [];
+
+  const pkgPojo = await fs.readJson(req.manifestPath);
+
+  if (!pkgPojo) {
+    prodDepMissingErrors.push(`${manifestPath} was parsed into a falsy POJO.`);
+  }
+
+  const { dependencies } = pkgPojo;
+  if (!dependencies || typeof dependencies !== "object") {
+    if (verbose) {
+      console.log("%s Skipping %s: no 'dependencies' key", fn, manifestPath);
+    }
+    return prodDepMissingErrors;
+  }
+  if (verbose) {
+    console.log("%s prod dependencies %s: %o", TAG, manifestPath, dependencies);
+  }
+
+  const depEntries = Object.entries(dependencies);
+  if (depEntries.length <= 0) {
+    if (verbose) {
+      console.log("%s Skipping %s: empty 'dependencies'", fn, manifestPath);
+    }
+    return prodDepMissingErrors;
+  }
+
+  const prodDepPkgs = depEntries.map(([k]) => k);
+
+  const { using } = depCheckResult;
+
+  const parserCache = new Map<string, ParseResult<BabelFile>>();
+
+  const seenAndIgnoredExt = new Set<string>();
+
+  const fileToAstMap = new Map<string, Array<ParseResult<BabelFile>>>();
+
+  const parseTasks = Object.entries(using).map(async ([k, v]) => {
+    const astsOfK = [];
+    for (let i = 0; i < v.length; i++) {
+      const filePath = v[i];
+      const supportedFileExtensions = [".ts", ".js"];
+      const fileExtension = path.extname(filePath);
+      if (!supportedFileExtensions.includes(fileExtension)) {
+        seenAndIgnoredExt.add(fileExtension);
+        continue;
+      }
+      const contentBuffer = await fs.readFile(filePath);
+      const contentStr = contentBuffer.toString("utf-8");
+
+      const cachedAst = parserCache.get(filePath);
+
+      const ast =
+        cachedAst ||
+        (await parse(contentStr, {
+          sourceType: "module",
+          sourceFilename: filePath,
+          attachComment: false,
+          plugins: [
+            "typescript",
+            "decorators-legacy",
+            "dynamicImport",
+            "importAttributes",
+            "importMeta",
+            "importReflection",
+            "deferredImportEvaluation",
+            "sourcePhaseImports",
+          ],
+        }));
+
+      parserCache.set(filePath, ast as ParseResult<BabelFile>);
+
+      astsOfK.push(ast);
+    }
+    fileToAstMap.set(k, astsOfK as Array<ParseResult<BabelFile>>);
+  });
+
+  if (seenAndIgnoredExt.size > 0) {
+    console.log("%s Seen+Ignored file extensions: %o", fn, seenAndIgnoredExt);
+  }
+
+  await Promise.all(parseTasks);
+
+  const missingProdDeps = Object.entries(using).filter(([k, v]) => {
+    // Is it being imported by code that is in a directory indicating prod use?
+    const pathPrefixMatch = v.some((x) =>
+      prodCodePathPrefixes.some((prefix) => x.startsWith(prefix)),
+    );
+    // Is it actually missing from the package.json#dependencies hash?
+    const isMissing = !prodDepPkgs.includes(k);
+
+    // We choose to ignore dependencies like @types/express or @types/* because
+    // we know that at runtime the Typescript compiler will delete those imports
+    const notIgnored = ignoredPkgNameRegExps.every((r: RegExp) => !r.test(k));
+
+    // If both are true we caught ourselves a missing production dependency that
+    // will crash at runtime if not rectified.
+    const isMissingProdDep = pathPrefixMatch && isMissing && notIgnored;
+
+    const asts = fileToAstMap.get(k);
+    if (!asts) {
+      const errorMessage = `${fn} Expected presence of parsed AST in map for dependency=${k}`;
+      throw new Error(errorMessage);
+    }
+
+    const andNotOnlyTypeImports = asts.some((ast) =>
+      ast.program.body
+        .filter((n): n is ImportDeclaration => {
+          return n.type === "ImportDeclaration";
+        })
+        .filter((n) => n.source.value === k)
+        .some((n) => n.importKind !== "type"),
+    );
+
+    return isMissingProdDep && andNotOnlyTypeImports;
+  });
+
+  if (!Array.isArray(missingProdDeps)) {
+    throw new Error(fn + "Expected missingProdDeps to be Array");
+  }
+
+  missingProdDeps.forEach(([k, v]) => {
+    const usageLocations = v.join("\n\t");
+    const errorMsg = `${TAG} ERROR - MISSING production dependency ${k} from ${manifestPath}. Found usage in \n\t${usageLocations}`;
+    prodDepMissingErrors.push(errorMsg);
+  });
+
+  return prodDepMissingErrors;
+}
+
 const nodePath = path.resolve(process.argv[1]);
 const modulePath = path.resolve(fileURLToPath(import.meta.url));
 const isRunningDirectlyViaCLI = nodePath === modulePath;
@@ -111,6 +313,7 @@ if (isRunningDirectlyViaCLI) {
   const { absolutePaths: pkgDirsToCheck } = await getAllPkgDirs();
 
   const req: ICheckMissingNodeDepsRequest = {
+    verbose: false,
     pkgDirsToCheck,
   };
   const [success, errors] = await checkMissingNodeDeps(req);

yarn.lock

@@ -2067,6 +2067,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/helper-string-parser@npm:^7.24.7":
+  version: 7.24.7
+  resolution: "@babel/helper-string-parser@npm:7.24.7"
+  checksum: 10/603d8d962bbe89907aa99a8f19a006759ab7b2464615f20a6a22e3e2e8375af37ddd0e5175c9e622e1c4b2d83607ffb41055a59d0ce34404502af30fde573a5c
+  languageName: node
+  linkType: hard
+
 "@babel/helper-validator-identifier@npm:^7.16.7":
   version: 7.16.7
   resolution: "@babel/helper-validator-identifier@npm:7.16.7"
@@ -2109,6 +2116,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/helper-validator-identifier@npm:^7.24.7":
+  version: 7.24.7
+  resolution: "@babel/helper-validator-identifier@npm:7.24.7"
+  checksum: 10/86875063f57361471b531dbc2ea10bbf5406e12b06d249b03827d361db4cad2388c6f00936bcd9dc86479f7e2c69ea21412c2228d4b3672588b754b70a449d4b
+  languageName: node
+  linkType: hard
+
 "@babel/helper-validator-option@npm:^7.16.7":
   version: 7.16.7
   resolution: "@babel/helper-validator-option@npm:7.16.7"
@@ -2341,6 +2355,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/parser@npm:7.24.7":
+  version: 7.24.7
+  resolution: "@babel/parser@npm:7.24.7"
+  bin:
+    parser: ./bin/babel-parser.js
+  checksum: 10/ef9ebce60e13db560ccc7af9235d460f6726bb7e23ae2d675098c1fc43d5249067be60d4118889dad33b1d4f85162cf66baf554719e1669f29bb20e71322568e
+  languageName: node
+  linkType: hard
+
 "@babel/parser@npm:^7.1.0, @babel/parser@npm:^7.14.7, @babel/parser@npm:^7.16.7, @babel/parser@npm:^7.17.3":
   version: 7.17.3
   resolution: "@babel/parser@npm:7.17.3"
@@ -4082,6 +4105,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@babel/types@npm:7.24.7":
+  version: 7.24.7
+  resolution: "@babel/types@npm:7.24.7"
+  dependencies:
+    "@babel/helper-string-parser": "npm:^7.24.7"
+    "@babel/helper-validator-identifier": "npm:^7.24.7"
+    to-fast-properties: "npm:^2.0.0"
+  checksum: 10/ad3c8c0d6fb4acb0bb74bb5b4bb849b181bf6185677ef9c59c18856c81e43628d0858253cf232f0eca806f02e08eff85a1d3e636a3e94daea737597796b0b430
+  languageName: node
+  linkType: hard
+
 "@babel/types@npm:^7.0.0, @babel/types@npm:^7.16.7, @babel/types@npm:^7.17.0, @babel/types@npm:^7.3.0, @babel/types@npm:^7.3.3, @babel/types@npm:^7.4.4":
   version: 7.17.0
   resolution: "@babel/types@npm:7.17.0"
@@ -9413,6 +9447,8 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@hyperledger/cactus@workspace:."
   dependencies:
+    "@babel/parser": "npm:7.24.7"
+    "@babel/types": "npm:7.24.7"
     "@bufbuild/buf": "npm:1.30.0"
     "@bufbuild/protobuf": "npm:1.8.0"
     "@bufbuild/protoc-gen-es": "npm:1.8.0"