test: use successRate for gateway conformance tests (#83)

* test: use successRate for gateway conformance tests

* chore: lint fix

* chore: allow overriding success rate via command line

* chore: lint fix

Author: SgtPooki

packages/gateway-conformance/src/conformance.spec.ts

@@ -15,8 +15,7 @@ interface TestConfig {
   spec?: string
   skip?: string[]
   run?: string[]
-  maxFailures: number
-  minimumSuccesses?: number
+  successRate: number
 }
 
 function getGatewayConformanceBinaryPath (): string {
@@ -43,224 +42,219 @@ function getConformanceTestArgs (name: string, gwcArgs: string[] = [], goTestArg
   ]
 }
 
+/**
+ * You can see what the latest success rates are by running the following command:
+ *
+ * ```
+ * cd ../../ && npm run build && cd packages/gateway-conformance && SUCCESS_RATE=100 npm run test -- --bail false
+ * ```
+ */
 const tests: TestConfig[] = [
   {
     name: 'TestMetadata',
     run: ['TestMetadata'],
-    maxFailures: 0,
-    minimumSuccesses: 1
+    successRate: 100
   },
   {
     name: 'TestDagPbConversion',
     run: ['TestDagPbConversion'],
-    maxFailures: 51,
-    minimumSuccesses: 14
+    successRate: 35.38
   },
   {
     name: 'TestPlainCodec',
     run: ['TestPlainCodec'],
-    maxFailures: 83,
-    minimumSuccesses: 15
+    successRate: 39.86
   },
   {
     name: 'TestPathing',
     run: ['TestPathing'],
-    maxFailures: 13,
-    minimumSuccesses: 0
+    successRate: 23.53
   },
   {
     name: 'TestDNSLinkGatewayUnixFSDirectoryListing',
     run: ['TestDNSLinkGatewayUnixFSDirectoryListing'],
-    maxFailures: 20,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestCors',
     run: ['TestCors'],
-    maxFailures: 11,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestGatewayJsonCbor',
     run: ['TestGatewayJsonCbor'],
-    maxFailures: 9,
-    minimumSuccesses: 0
+    successRate: 44.44
   },
   // currently results in an infinite loop without verified-fetch stopping the request whether sessions are enabled or not.
   // {
   //   name: 'TestNativeDag',
   //   run: ['TestNativeDag'],
-  //   maxFailures: 2,
-  //   minimumSuccesses: 0
+  //   successRate: 100
   // },
   {
     name: 'TestGatewayJSONCborAndIPNS',
     run: ['TestGatewayJSONCborAndIPNS'],
-    maxFailures: 25,
-    minimumSuccesses: 8
+    successRate: 24.24
   },
   {
     name: 'TestGatewayIPNSPath',
     run: ['TestGatewayIPNSPath'],
-    maxFailures: 8,
-    minimumSuccesses: 3
+    successRate: 27.27
   },
   {
     name: 'TestRedirectCanonicalIPNS',
     run: ['TestRedirectCanonicalIPNS'],
-    maxFailures: 7,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestGatewayBlock',
     run: ['TestGatewayBlock'],
-    maxFailures: 25,
-    minimumSuccesses: 4
+    successRate: 37.93
   },
   {
     name: 'TestTrustlessRawRanges',
     run: ['TestTrustlessRawRanges'],
-    maxFailures: 5,
-    minimumSuccesses: 7
+    successRate: 75
   },
   {
     name: 'TestTrustlessRaw',
     run: ['TestTrustlessRaw'],
-    maxFailures: 29,
-    minimumSuccesses: 7
+    successRate: 55.56
   },
   {
     name: 'TestGatewayIPNSRecord',
     run: ['TestGatewayIPNSRecord'],
-    maxFailures: 23,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestTrustlessCarOrderAndDuplicates',
     run: ['TestTrustlessCarOrderAndDuplicates'],
-    maxFailures: 26,
-    minimumSuccesses: 3
+    successRate: 13.79
   },
   // times out
   // {
   //   name: 'TestTrustlessCarEntityBytes',
   //   run: ['TestTrustlessCarEntityBytes'],
-  //   maxFailures: 122,
-  //   minimumSuccesses: 55
+  //   successRate: 100
   // },
   {
     name: 'TestTrustlessCarDagScopeAll',
     run: ['TestTrustlessCarDagScopeAll'],
-    maxFailures: 23,
-    minimumSuccesses: 10
+    successRate: 36.36
   },
   {
     name: 'TestTrustlessCarDagScopeEntity',
     run: ['TestTrustlessCarDagScopeEntity'],
-    maxFailures: 56,
-    minimumSuccesses: 25
+    successRate: 34.57
   },
   {
     name: 'TestTrustlessCarDagScopeBlock',
     run: ['TestTrustlessCarDagScopeBlock'],
-    maxFailures: 34,
-    minimumSuccesses: 15
+    successRate: 34.69
   },
   {
     name: 'TestTrustlessCarPathing',
     run: ['TestTrustlessCarPathing'],
-    maxFailures: 45,
-    minimumSuccesses: 20
+    successRate: 33.85
   },
   {
     name: 'TestSubdomainGatewayDNSLinkInlining',
     run: ['TestSubdomainGatewayDNSLinkInlining'],
-    maxFailures: 41,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestGatewaySubdomainAndIPNS',
     run: ['TestGatewaySubdomainAndIPNS'],
-    maxFailures: 95,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestGatewaySubdomains',
     run: ['TestGatewaySubdomains'],
-    maxFailures: 279,
-    minimumSuccesses: 0
+    successRate: 7.17
   },
   // times out
   // {
   //   name: 'TestUnixFSDirectoryListingOnSubdomainGateway',
   //   run: ['TestUnixFSDirectoryListingOnSubdomainGateway'],
-  //   maxFailures: 39,
-  //   minimumSuccesses: 0
+  //   successRate: 100
   // },
   {
     name: 'TestRedirectsFileWithIfNoneMatchHeader',
     run: ['TestRedirectsFileWithIfNoneMatchHeader'],
-    maxFailures: 15,
-    minimumSuccesses: 0
+    successRate: 0
   },
   {
     name: 'TestRedirectsFileSupportWithDNSLink',
     run: ['TestRedirectsFileSupportWithDNSLink'],
-    maxFailures: 17,
-    minimumSuccesses: 6
+    successRate: 26.09
   },
   {
     name: 'TestRedirectsFileSupport',
     run: ['TestRedirectsFileSupport'],
-    maxFailures: 252,
-    minimumSuccesses: 6
+    successRate: 2.33
   },
   {
     name: 'TestPathGatewayMiscellaneous',
     run: ['TestPathGatewayMiscellaneous'],
-    maxFailures: 3,
-    minimumSuccesses: 0
+    successRate: 100
   },
   {
     name: 'TestGatewayUnixFSFileRanges',
     run: ['TestGatewayUnixFSFileRanges'],
-    maxFailures: 10,
-    minimumSuccesses: 5
+    successRate: 40
   },
   {
     name: 'TestGatewaySymlink',
     run: ['TestGatewaySymlink'],
-    maxFailures: 9,
-    minimumSuccesses: 0
+    successRate: 33.33
   },
   {
     name: 'TestGatewayCacheWithIPNS',
     run: ['TestGatewayCacheWithIPNS'],
-    maxFailures: 27,
-    minimumSuccesses: 15
+    successRate: 35.71
   },
   // times out
   // {
   //   name: 'TestGatewayCache',
   //   run: ['TestGatewayCache'],
-  //   maxFailures: 71,
-  //   minimumSuccesses: 23
+  //   successRate: 100
   // },
   // times out
   // {
   //   name: 'TestUnixFSDirectoryListing',
   //   run: ['TestUnixFSDirectoryListing'],
-  //   maxFailures: 50,
-  //   minimumSuccesses: 0
+  //   successRate: 100
   // },
   {
     name: 'TestTar',
     run: ['TestTar'],
-    maxFailures: 16,
-    minimumSuccesses: 8
+    successRate: 50
   }
 ]
 
+async function getReportDetails (path: string): Promise<{ failureCount: number, successCount: number, successRate: number }> {
+  let failureCount = 0
+  let successCount = 0
+
+  // parse the newline delimited JSON report at gwc-report-${name}.json and count the number of "PASS:" and "FAIL:" lines
+  const report = await readFile(path, 'utf8')
+  const lines = report.split('\n')
+  for (const line of lines) {
+    if (line.includes('--- FAIL:')) {
+      failureCount++
+    } else if (line.includes('--- PASS:')) {
+      successCount++
+    }
+  }
+  const successRate = Number.parseFloat(((successCount / (successCount + failureCount)) * 100).toFixed(2))
+
+  return {
+    failureCount,
+    successCount,
+    successRate
+  }
+}
+
 describe('@helia/verified-fetch - gateway conformance', function () {
   before(async () => {
     if (process.env.KUBO_GATEWAY == null) {
@@ -325,10 +319,11 @@ describe('@helia/verified-fetch - gateway conformance', function () {
       }
     })
 
-    tests.forEach(({ name, spec, skip, run, maxFailures, minimumSuccesses }) => {
+    tests.forEach(({ name, spec, skip, run, successRate: minSuccessRate }) => {
       const log = logger.forComponent(name)
+      const expectedSuccessRate = process.env.SUCCESS_RATE != null ? Number.parseFloat(process.env.SUCCESS_RATE) : minSuccessRate
 
-      it(`has no more than ${maxFailures} failing tests for ${name}`, async function () {
+      it(`${name} has a success rate of at least ${expectedSuccessRate}%`, async function () {
         const { stderr, stdout } = await execa(binaryPath, getConformanceTestArgs(name,
           [
             ...(spec != null ? ['--specs', spec] : [])
@@ -342,22 +337,8 @@ describe('@helia/verified-fetch - gateway conformance', function () {
         log(stdout)
         log.error(stderr)
 
-        let failureCount = 0
-        let successCount = 0
-
-        // parse the newline delimited JSON report at gwc-report-${name}.json and count the number of "PASS:" and "FAIL:" lines
-        const report = await readFile(`gwc-report-${name}.json`, 'utf8')
-        const lines = report.split('\n')
-        for (const line of lines) {
-          if (line.includes('--- FAIL:')) {
-            failureCount++
-          } else if (line.includes('--- PASS:')) {
-            successCount++
-          }
-        }
-
-        expect(failureCount).to.be.lessThanOrEqual(maxFailures)
-        expect(successCount).to.be.greaterThanOrEqual(minimumSuccesses ?? 0)
+        const { successRate } = await getReportDetails(`gwc-report-${name}.json`)
+        expect(successRate).to.be.greaterThanOrEqual(expectedSuccessRate)
       })
     })
 
@@ -382,22 +363,12 @@ describe('@helia/verified-fetch - gateway conformance', function () {
 
       log(stdout)
       log.error(stderr)
-      let failureCount = 0
-      let successCount = 0
 
-      // parse the newline delimited JSON report at gwc-report-${name}.json and count the number of "PASS:" and "FAIL:" lines
-      const report = await readFile('gwc-report-all.json', 'utf8')
-      const lines = report.split('\n')
-      for (const line of lines) {
-        if (line.includes('--- FAIL:')) {
-          failureCount++
-        } else if (line.includes('--- PASS:')) {
-          successCount++
-        }
-      }
-      // CI has 1134 failures, but I get 1129 locally.
+      const { failureCount, successCount, successRate } = await getReportDetails('gwc-report-all.json')
+
       expect(failureCount).to.be.lessThanOrEqual(1134)
       expect(successCount).to.be.greaterThanOrEqual(262)
+      expect(successRate).to.be.greaterThanOrEqual(18.77)
     })
   })
 })