feat: node signature verification for queries

.github/workflows/e2e-tests.yml

@@ -42,6 +42,10 @@ jobs:
         run: |
           dfx start --background
 
+      - name: setup
+        id: setup
+        run: npm run setup --workspaces --if-present
+
       - run: npm run e2e --workspaces --if-present
         env:
           CI: true

.github/workflows/mitm.yml

@@ -7,6 +7,7 @@ on:
       - reopened
       - edited
       - synchronize
+  workflow_dispatch:
 
 jobs:
   test:
@@ -37,8 +38,8 @@ jobs:
 
       - uses: actions/setup-python@v2
         with:
-          python-version: '3.8'
-      - run: pip3 install mitmproxy~=8.0.0
+          python-version: '3.11'
+      - run: pip3 install mitmproxy~=10.0.0
 
       - run: echo y | sh -ci "$(curl -fsSL https://sdk.dfinity.org/install.sh)"
 
@@ -51,17 +52,15 @@ jobs:
         id: mitmdump
         run: |
           set -ex
-          mitmdump -p 8888 --mode reverse:http://127.0.0.1:4943 \
+          mitmdump -p 8888 --mode reverse:https://icp-api.io \
             --modify-headers '/~s/Transfer-Encoding/' \
             --modify-body '/~s/Hello/Hullo' \
             &
-          sleep 5
+          sleep 20
 
       - name: mitm e2e
         env:
           CI: true
-          REPLICA_PORT: 8888
-          MITM: true
         run: npm run mitm --workspaces --if-present
 
   aggregate:

canister_ids.json

@@ -1,4 +1,7 @@
 {
+  "counter": {
+    "ic": "tnnnb-2yaaa-aaaab-qaiiq-cai"
+  },
   "docs": {
     "ic": "erxue-5aaaa-aaaab-qaagq-cai"
   }

demos/sample-javascript/README.md

@@ -6,4 +6,4 @@
 
 1. Ensure all dependencies are installed: `npm install`
 2. Run the development server `npm run develop`
-3. Visit the running site at http://localhost:8080
+3. Visit the running site at http://127.0.0.1:8080

dfx.json

@@ -3,6 +3,10 @@
     "docs": {
       "type": "assets",
       "source": ["docs/generated"]
+    },
+    "counter": {
+      "type": "motoko",
+      "main": "e2e/node/canisters/counter.mo"
     }
   }
 }

docs/generated/changelog.html

@@ -12,6 +12,27 @@ <h1>Agent-JS Changelog</h1>
     <section>
       <h2>Version x.x.x</h2>
       <ul>
+        <ul>
+          <strong>feat!: node signature verification</strong
+          ><br />
+          This feature includes additional changes in support of testing and releasing the feature:
+          <br />
+          <li>Mainnet e2e tests for queries and calls</li>
+          <li>published counter canister</li>
+          <li>using vitest for e2e tests instead of Jest</li>
+          <li>
+            New HttpAgent option - verifyQuerySignatures. Defaults to true, but allows you to opt
+            out of verification. Useful for testing against older replica versions
+          </li>
+          <li>Introducing ed25519 logic to agent for validating node signatures</li>
+          <li>Standardizing around @noble/curves instead of tweetnacl in @dfinity/identity</li>
+          <li>
+            new export - hashOfMap from agent, matching the naming used in the interface
+            specification
+          </li>
+          <li>new unit tests</li>
+          <li>new Verify export on ed25519 because why not</li>
+        </ul>
         <li>feat: retry logic will catch and retry for thrown errors</li>
         <li>
           feat!: adds certificate logic to decode subnet and node key paths from the hashtree.

e2e/browser/.proxyrc

@@ -1,5 +1,5 @@
 {
   "/api": {
-    "target": "http://localhost:4943/",
+    "target": "http://127.0.0.1:4943/"
   }
 }

e2e/browser/cypress/e2e/ecdsa.cy.js

@@ -2,11 +2,19 @@ import { ECDSAKeyIdentity } from '@dfinity/identity';
 import { get, set } from 'idb-keyval';
 import { createActor } from '../utils/actor';
 import ids from '../../.dfx/local/canister_ids.json';
+import fetchPolyfill from 'isomorphic-fetch';
 const canisterId = ids.whoami.local;
 
 const setup = async () => {
   const identity1 = await ECDSAKeyIdentity.generate();
-  const whoami1 = createActor(ids.whoami.local, { agentOptions: { identity: identity1 } });
+  const whoami1 = createActor(ids.whoami.local, {
+    agentOptions: {
+      verifyQuerySignatures: false,
+      identity: identity1,
+      fetch: fetchPolyfill,
+      host: 'http://127.0.0.1:4943/',
+    },
+  });
 
   const principal1 = await whoami1.whoami();
 
@@ -34,7 +42,14 @@ describe('ECDSAKeyIdentity tests with SubtleCrypto', () => {
 
       const identity2 = await ECDSAKeyIdentity.fromKeyPair(storedKeyPair);
 
-      const whoami2 = createActor(canisterId, { agentOptions: { identity: identity2 } });
+      const whoami2 = createActor(canisterId, {
+        agentOptions: {
+          verifyQuerySignatures: false,
+          identity: identity2,
+          fetchPolyfill,
+          host: 'http://127.0.0.1:4943/',
+        },
+      });
 
       const principal2 = await whoami2.whoami();
 

e2e/browser/package.json

@@ -4,9 +4,10 @@
   "version": "0.19.3",
   "scripts": {
     "ci": "npm run e2e",
-    "pree2e": "dfx deploy; dfx generate; pm2 --name parcel start npm -- start",
-    "e2e": "cypress run",
-    "poste2e": "pm2 delete 0",
+    "setup": "dfx deploy; dfx generate; pm2 --name parcel start npm -- start",
+    "cypress": "cypress run",
+    "e2e": "npm run cypress",
+    "poste2e": "pm2 kill",
     "eslint:fix": "npm run lint -- --fix",
     "eslint": "eslint --ext '.js,.jsx,.ts,.tsx' cypress *.js",
     "lint": "npm run eslint",
@@ -15,7 +16,7 @@
     "publish:release": "",
     "test:coverage": "",
     "test": "",
-    "start": "parcel --watch-for-stdin src/index.html"
+    "start": "parcel src/index.html"
   },
   "devDependencies": {
     "@types/node": "^18.0.6",
@@ -28,7 +29,6 @@
   },
   "dependencies": {
     "@dfinity/agent": "^0.19.3",
-    "@dfinity/authentication": "^0.14.1",
     "@dfinity/identity": "^0.19.3",
     "@dfinity/principal": "^0.19.3",
     "idb-keyval": "^6.2.0"

e2e/node/basic/assets.test.ts

@@ -1,12 +1,10 @@
-/**
- * @jest-environment node
- */
 import { existsSync, readFileSync, unlinkSync } from 'fs';
 import path from 'path';
 import agent from '../utils/agent';
 import { Actor } from '@dfinity/agent';
 import { Principal } from '@dfinity/principal';
 import { AssetManager } from '@dfinity/assets';
+import { test, describe, it, expect, beforeAll, afterEach } from 'vitest';
 
 /**
  * Create (pseudo) random bytes Readable
@@ -43,7 +41,6 @@ const testFile = {
   target: path.join(__dirname, '../package_copy.json'),
 };
 
-jest.setTimeout(100000);
 describe('assets', () => {
   let canisterId: Principal;
 
@@ -84,9 +81,17 @@ describe('assets', () => {
     }
   });
 
-  it('store, get and delete 1MB asset (single chunk)', () => testRandomBytes('1MB.bin', 1000000));
+  it(
+    'store, get and delete 1MB asset (single chunk)',
+    () => testRandomBytes('1MB.bin', 1000000),
+    100000,
+  );
 
-  it('store, get and delete 3MB asset (multiple chunk)', () => testRandomBytes('3MB.bin', 3000000));
+  it(
+    'store, get and delete 3MB asset (multiple chunk)',
+    () => testRandomBytes('3MB.bin', 3000000),
+    100000,
+  );
 
   it('batch process assets and verify asset list', async () => {
     const assetManager = new AssetManager({ canisterId, agent: await agent });

e2e/node/basic/basic.test.ts

@@ -5,6 +5,7 @@ import { ActorMethod, Certificate, getManagementCanister } from '@dfinity/agent'
 import { IDL } from '@dfinity/candid';
 import { Principal } from '@dfinity/principal';
 import agent from '../utils/agent';
+import { test, expect } from 'vitest';
 
 test('read_state', async () => {
   const resolvedAgent = await agent;

e2e/node/basic/canisterStatus.test.ts

@@ -1,15 +1,16 @@
 import { CanisterStatus, HttpAgent } from '@dfinity/agent';
 import { Principal } from '@dfinity/principal';
 import counter from '../canisters/counter';
+import { makeAgent } from '../utils/agent';
+import { describe, it, afterEach, expect } from 'vitest';
 
-jest.setTimeout(30_000);
 afterEach(async () => {
   await Promise.resolve();
 });
 describe('canister status', () => {
   it('should fetch successfully', async () => {
     const counterObj = await (await counter)();
-    const agent = new HttpAgent({ host: `http://localhost:${process.env.REPLICA_PORT}` });
+    const agent = await makeAgent();
     await agent.fetchRootKey();
     const request = await CanisterStatus.request({
       canisterId: Principal.from(counterObj.canisterId),
@@ -21,7 +22,10 @@ describe('canister status', () => {
   });
   it('should throw an error if fetchRootKey has not been called', async () => {
     const counterObj = await (await counter)();
-    const agent = new HttpAgent({ host: `http://localhost:${process.env.REPLICA_PORT}` });
+    const agent = new HttpAgent({
+      host: `http://127.0.0.1:${process.env.REPLICA_PORT ?? 4943}`,
+      verifyQuerySignatures: false,
+    });
     const shouldThrow = async () => {
       // eslint-disable-next-line no-useless-catch
       try {

e2e/node/basic/counter.test.ts

@@ -1,9 +1,6 @@
-/**
- * @jest-environment node
- */
 import counterCanister, { noncelessCanister, createActor } from '../canisters/counter';
+import { it, expect, describe, vi } from 'vitest';
 
-jest.setTimeout(40000);
 describe('counter', () => {
   it('should greet', async () => {
     const { actor: counter } = await counterCanister();
@@ -12,7 +9,7 @@ describe('counter', () => {
     } catch (error) {
       console.error(error);
     }
-  });
+  }, 40000);
   it('should submit distinct requests with nonce by default', async () => {
     const { actor: counter } = await counterCanister();
     const values = await Promise.all(new Array(4).fill(undefined).map(() => counter.inc_read()));
@@ -23,7 +20,7 @@ describe('counter', () => {
     // Sets of unique results should be the same length
     expect(set1.size).toBe(values.length);
     expect(set2.size).toEqual(values2.length);
-  });
+  }, 40000);
   it('should submit duplicate requests if nonce is disabled', async () => {
     const { actor: counter } = await noncelessCanister();
     const values = await Promise.all(new Array(4).fill(undefined).map(() => counter.inc_read()));
@@ -32,7 +29,7 @@ describe('counter', () => {
     const set2 = new Set(values2);
 
     expect(set1.size < values.length || set2.size < values2.length).toBe(true);
-  });
+  }, 40000);
   it('should increment', async () => {
     const { actor: counter } = await noncelessCanister();
 
@@ -41,13 +38,12 @@ describe('counter', () => {
     expect(Number(await counter.read())).toEqual(1);
     await counter.inc();
     expect(Number(await counter.read())).toEqual(2);
-  });
+  }, 40000);
 });
 describe('retrytimes', () => {
   it('should retry after a failure', async () => {
-    jest.spyOn(console, 'warn').mockImplementation();
     let count = 0;
-    const fetchMock = jest.fn(function (...args) {
+    const fetchMock = vi.fn(function (...args) {
       if (count <= 1) {
         count += 1;
         return new Response('Test error - ignore', {
@@ -68,5 +64,5 @@ describe('retrytimes', () => {
     } catch (error) {
       console.error(error);
     }
-  });
+  }, 40000);
 });