feat: enhance the solidity test artifacts discovery

v-next/hardhat/package.json

@@ -81,11 +81,12 @@
     "typescript-eslint": "7.7.1"
   },
   "dependencies": {
-    "@nomicfoundation/solidity-analyzer": "^0.1.0",
-    "@ignored/hardhat-vnext-zod-utils": "workspace:^3.0.0-next.3",
-    "@ignored/hardhat-vnext-utils": "workspace:^3.0.0-next.3",
-    "@ignored/hardhat-vnext-errors": "workspace:^3.0.0-next.3",
     "@ignored/edr": "0.6.2-alpha.0",
+    "@ignored/hardhat-vnext-errors": "workspace:^3.0.0-next.3",
+    "@ignored/hardhat-vnext-utils": "workspace:^3.0.0-next.3",
+    "@ignored/hardhat-vnext-zod-utils": "workspace:^3.0.0-next.3",
+    "@nomicfoundation/slang": "^0.18.2",
+    "@nomicfoundation/solidity-analyzer": "^0.1.0",
     "@sentry/node": "^5.18.1",
     "adm-zip": "^0.4.16",
     "chalk": "^5.3.0",

v-next/hardhat/src/internal/builtin-plugins/solidity-test/helpers.ts

@@ -9,6 +9,10 @@ import type {
 
 import { runSolidityTests } from "@ignored/edr";
 import { HardhatError } from "@ignored/hardhat-vnext-errors";
+import { exists, readUtf8File } from "@ignored/hardhat-vnext-utils/fs";
+import { resolveFromRoot } from "@ignored/hardhat-vnext-utils/path";
+import { NonterminalKind, TerminalKind } from "@nomicfoundation/slang/cst";
+import { Parser } from "@nomicfoundation/slang/parser";
 
 /**
  * Run all the given solidity tests and returns the whole results after finishing.
@@ -50,13 +54,11 @@ export async function runAllSolidityTests(
   });
 }
 
-export async function buildSolidityTestsInput(
+export async function getArtifacts(
   hardhatArtifacts: ArtifactsManager,
-  isTestArtifact: (artifact: Artifact) => boolean = () => true,
-): Promise<{ artifacts: Artifact[]; testSuiteIds: ArtifactId[] }> {
+): Promise<Artifact[]> {
   const fqns = await hardhatArtifacts.getAllFullyQualifiedNames();
   const artifacts: Artifact[] = [];
-  const testSuiteIds: ArtifactId[] = [];
 
   for (const fqn of fqns) {
     const hardhatArtifact = await hardhatArtifacts.readArtifact(fqn);
@@ -85,10 +87,83 @@ export async function buildSolidityTestsInput(
 
     const artifact = { id, contract };
     artifacts.push(artifact);
-    if (isTestArtifact(artifact)) {
-      testSuiteIds.push(artifact.id);
+  }
+
+  return artifacts;
+}
+
+export async function isTestArtifact(
+  root: string,
+  artifact: Artifact,
+): Promise<boolean> {
+  const { name, source, solcVersion } = artifact.id;
+
+  if (!source.endsWith(".t.sol")) {
+    return false;
+  }
+
+  const sourcePath = resolveFromRoot(root, source);
+  const sourceExists = await exists(sourcePath);
+
+  if (!sourceExists) {
+    return false;
+  }
+
+  const content = await readUtf8File(sourcePath);
+  const parser = Parser.create(solcVersion);
+  const cursor = parser
+    .parse(NonterminalKind.SourceUnit, content)
+    .createTreeCursor();
+
+  while (
+    cursor.goToNextNonterminalWithKind(NonterminalKind.ContractDefinition)
+  ) {
+    const nameCursor = cursor.spawn();
+    if (!nameCursor.goToNextTerminalWithKind(TerminalKind.Identifier)) {
+      continue;
+    }
+    if (nameCursor.node.unparse() !== name) {
+      continue;
+    }
+
+    const abstractCursor = cursor.spawn();
+    if (abstractCursor.goToNextTerminalWithKind(TerminalKind.AbstractKeyword)) {
+      return false;
+    }
+
+    const functionCursor = cursor.spawn();
+
+    while (
+      functionCursor.goToNextNonterminalWithKind(
+        NonterminalKind.FunctionDefinition,
+      )
+    ) {
+      const functionNameCursor = functionCursor.spawn();
+      if (
+        !functionNameCursor.goToNextTerminalWithKind(TerminalKind.Identifier)
+      ) {
+        continue;
+      }
+
+      const functionName = functionNameCursor.node.unparse();
+      if (
+        functionName.startsWith("test") ||
+        functionName.startsWith("invariant")
+      ) {
+        const publicCursor = functionCursor.spawn();
+        if (publicCursor.goToNextTerminalWithKind(TerminalKind.PublicKeyword)) {
+          return true;
+        }
+
+        const externalCursor = functionCursor.spawn();
+        if (
+          externalCursor.goToNextTerminalWithKind(TerminalKind.ExternalKeyword)
+        ) {
+          return true;
+        }
+      }
     }
   }
 
-  return { artifacts, testSuiteIds };
+  return false;
 }

v-next/hardhat/src/internal/builtin-plugins/solidity-test/task-action.ts

@@ -2,7 +2,11 @@ import type { NewTaskActionFunction } from "../../../types/tasks.js";
 
 import { spec } from "node:test/reporters";
 
-import { buildSolidityTestsInput, runAllSolidityTests } from "./helpers.js";
+import {
+  getArtifacts,
+  isTestArtifact,
+  runAllSolidityTests,
+} from "./helpers.js";
 
 const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
   await hre.tasks.getTask("compile").run({ quiet: false });
@@ -16,19 +20,16 @@ const runSolidityTests: NewTaskActionFunction = async (_arguments, hre) => {
   let totalTests = 0;
   let failedTests = 0;
 
-  const { artifacts, testSuiteIds } = await buildSolidityTestsInput(
-    hre.artifacts,
-    (artifact) => {
-      const sourceName = artifact.id.source;
-      const isTestArtifact =
-        sourceName.endsWith(".t.sol") &&
-        sourceName.startsWith("contracts/") &&
-        !sourceName.startsWith("contracts/forge-std/") &&
-        !sourceName.startsWith("contracts/ds-test/");
-
-      return isTestArtifact;
-    },
-  );
+  const artifacts = await getArtifacts(hre.artifacts);
+  const testSuiteIds = (
+    await Promise.all(
+      artifacts.map(async (artifact) => {
+        if (await isTestArtifact(hre.config.paths.root, artifact)) {
+          return artifact.id;
+        }
+      }),
+    )
+  ).filter((artifact) => artifact !== undefined);
 
   const config = {
     projectRoot: hre.config.paths.root,

v-next/hardhat/test/internal/builtin-plugins/solidity-test/helpers.ts

@@ -0,0 +1,98 @@
+import type { Artifact } from "@ignored/edr";
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+
+import { isTestArtifact } from "../../../../src/internal/builtin-plugins/solidity-test/helpers.js";
+
+const testCases = [
+  {
+    contract: "Abstract",
+    expected: false,
+  },
+  {
+    contract: "NoTest",
+    expected: false,
+  },
+  {
+    contract: "PublicTest",
+    expected: true,
+  },
+  {
+    contract: "ExternalTest",
+    expected: true,
+  },
+  {
+    contract: "PrivateTest",
+    expected: false,
+  },
+  {
+    contract: "InternalTest",
+    expected: false,
+  },
+  {
+    contract: "PublicInvariant",
+    expected: true,
+  },
+  {
+    contract: "ExternalInvariant",
+    expected: true,
+  },
+  {
+    contract: "PrivateInvariant",
+    expected: false,
+  },
+  {
+    contract: "InternalInvariant",
+    expected: false,
+  },
+];
+
+describe.only("isTestArtifact", () => {
+  for (const { contract, expected } of testCases) {
+    it(`should return ${expected} for the ${contract} contract`, async () => {
+      const artifact: Artifact = {
+        id: {
+          name: contract,
+          source: `test-fixtures/Test.t.sol`,
+          solcVersion: "0.8.20",
+        },
+        contract: {
+          abi: "",
+        },
+      };
+      const actual = await isTestArtifact(import.meta.dirname, artifact);
+      assert.equal(actual, expected);
+    });
+  }
+
+  it("should return false if a file does not exist", async () => {
+    const artifact: Artifact = {
+      id: {
+        name: "Contract",
+        source: `test-fixtures/NonExistent.t.sol`,
+        solcVersion: "0.8.20",
+      },
+      contract: {
+        abi: "",
+      },
+    };
+    const actual = await isTestArtifact(import.meta.dirname, artifact);
+    assert.equal(actual, false);
+  });
+
+  it("should return false if the file has the wrong extension", async () => {
+    const artifact: Artifact = {
+      id: {
+        name: "Contract",
+        source: `test-fixtures/WrongExtension.sol`,
+        solcVersion: "0.8.20",
+      },
+      contract: {
+        abi: "",
+      },
+    };
+    const actual = await isTestArtifact(import.meta.dirname, artifact);
+    assert.equal(actual, false);
+  });
+});

v-next/hardhat/test/internal/builtin-plugins/solidity-test/test-fixtures/Test.t.sol

@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: MIT
+pragma solidity 0.8.20;
+
+abstract contract Abstract {
+  function testPublic() public {}
+  function testExternal() external {}
+  function invariantPublic() public {}
+  function invariantExternal() external {}
+}
+
+contract NoTest {}
+
+contract PublicTest {
+  function test() public {}
+}
+
+contract ExternalTest {
+  function test() external {}
+}
+
+contract PrivateTest {
+  function test() private {}
+}
+
+contract InternalTest {
+  function test() internal {}
+}
+
+contract PublicInvariant {
+  function invariant() public {}
+}
+
+contract ExternalInvariant {
+  function invariant() external {}
+}
+
+contract PrivateInvariant {
+  function invariant() private {}
+}
+
+contract InternalInvariant {
+  function invariant() internal {}
+}