Archiver (#41)

* refactor: renamed data-archiver to rollup-archiver + initial setup

* chore: updated deps

* docs: inital RollupSource interface documentation

* feat: setting up RollupArchiver class

* feat: Add RollupBlockData

* temporarily added some deps from Aztec Connect

* feat: making sync loop work

* feat: status interface + implementation

* chore: updated jest config to avoid depreciation warnings

* test: archiver test setup

* docs: fixed naming

* feat: sync loop using viem

* refactor: naming

* refactor: improved naming 2

* cleanup

* installed types

* refactor: better naming + matching yeets to blocks

* docs: fixes

* refactor: renamed Data Archiver to Archiver

* docs: readme

* refactor: block number as number

* docs: issues in readme

* ordering

* feat: fetching latest block num from contract

* docs: removed redundant space

* test: Archiver

* chore: less noisy logs

* chore: reenabled `noImplicitAny` rule

* Update bootstrap.sh

* docs: fix

---------

Co-authored-by: LHerskind <lasse.herskind@gmail.com>
Co-authored-by: ludamad <adam.domurad@gmail.com>

README.md

@@ -3,11 +3,11 @@
 The Aztec 3 system consists of the following sub projects.
 
 - `acir-simulator`
+- `archiver`
 - `aztec-cli`
 - `aztec.js`
 - `barretenberg.js`
 - `circuit.js`
-- `data-archiver`
 - `ethereum.js`
 - `kernel-simulator`
 - `key-store`

bootstrap.sh

@@ -48,7 +48,7 @@ PROJECTS=(
   "yarn-project/aztec.js:yarn build"
   # "yarn-project/barretenberg.js:yarn build"
   # "yarn-project/circuit.js:yarn build"
-  # "yarn-project/data-archiver:yarn build"
+  "yarn-project/archiver:yarn build"
   # "yarn-project/ethereum.js:yarn build"
   # "yarn-project/kernel-simulator:yarn build"
   "yarn-project/key-store:yarn build"

build_manifest.json

@@ -45,6 +45,13 @@
     "rebuildPatterns": ["^yarn-project/acir-simulator/"],
     "dependencies": ["yarn-project-base"]
   },
+  "archiver": {
+    "buildDir": "yarn-project",
+    "projectDir": "yarn-project/archiver",
+    "dockerfile": "archiver/Dockerfile",
+    "rebuildPatterns": ["^yarn-project/archiver/"],
+    "dependencies": ["yarn-project-base"]
+  },
   "aztec-cli": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/aztec-cli",
@@ -73,13 +80,6 @@
     "rebuildPatterns": ["^yarn-project/circuit.js/"],
     "dependencies": ["yarn-project-base"]
   },
-  "data-archiver": {
-    "buildDir": "yarn-project",
-    "projectDir": "yarn-project/data-archiver",
-    "dockerfile": "data-archiver/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/data-archiver/"],
-    "dependencies": ["yarn-project-base"]
-  },
   "end-to-end": {
     "buildDir": "yarn-project",
     "projectDir": "yarn-project/end-to-end",

build_manifest.sh

@@ -12,11 +12,11 @@ PROJECTS=(
   # l1-contracts:l1-contracts
   # yarn-project-base:yarn-project
   # acir-simulator:yarn-project
+  # archiver:yarn-project
   # aztec-cli:yarn-project
   # aztec.js:yarn-project
   # barretenberg.js:yarn-project
   # circuit.js:yarn-project
-  # data-archiver:yarn-project
   # end-to-end:yarn-project
   # ethereum.js:yarn-project
   # kernel-simulator:yarn-project

yarn-project/data-archiver/Dockerfile → yarn-project/archiver/Dockerfile

@@ -1,14 +1,14 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base AS builder
 
-COPY data-archiver data-archiver
-WORKDIR /usr/src/yarn-project/data-archiver
+COPY archiver archiver
+WORKDIR /usr/src/yarn-project/archiver
 RUN yarn build && yarn formatting && yarn test
 
 # Prune dev dependencies. See comment in base image.
 RUN yarn cache clean
 RUN yarn workspaces focus --production > /dev/null
 
 FROM node:18-alpine
-COPY --from=builder /usr/src/yarn-project/data-archiver /usr/src/yarn-project/data-archiver
-WORKDIR /usr/src/yarn-project/data-archiver
+COPY --from=builder /usr/src/yarn-project/archiver /usr/src/yarn-project/archiver
+WORKDIR /usr/src/yarn-project/archiver
 ENTRYPOINT ["yarn"]

yarn-project/archiver/README.md

@@ -0,0 +1,9 @@
+# Archiver
+To run:
+1. Run `anvil`,
+2. in the aztec3-l1-contracts repo check out my branch `janb/archiver-test-data`,
+3. deploy the contracts and generate initial activity with: `forge script --fork-url "http://127.0.0.1:8545/" --ffi GenerateActivityTest --sig "testGenerateActivity()" --private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 --broadcast`
+4. in this repository run `yarn start:dev` (Note: this repo is currently messy and eslint will not allow it to be built with `yarn start`)
+
+Relevant unresolved issues:
+https://stackoverflow.com/questions/75739742/why-is-eslint-complaining-about-inheritdoc-tag

yarn-project/data-archiver/package.json → yarn-project/archiver/package.json

@@ -1,44 +1,47 @@
 {
-  "name": "@aztec/data-archiver",
+  "name": "@aztec/archiver",
   "version": "0.0.0",
   "type": "module",
   "exports": "./dest/index.js",
   "typedoc": {
     "entryPoint": "./src/index.ts",
-    "displayName": "Data Archiver",
+    "displayName": "Archiver",
     "tsconfig": "./tsconfig.dest.json"
   },
   "scripts": {
     "build": "yarn clean && yarn formatting && tsc -b tsconfig.dest.json",
     "build:dev": "tsc -b tsconfig.dest.json --watch",
+    "start": "node ./dest",
+    "start:dev": "tsc-watch -p tsconfig.dest.json --onSuccess 'yarn start'",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "formatting": "run -T prettier --check ./src && run -T eslint --max-warnings 0 ./src",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests"
   },
   "jest": {
     "preset": "ts-jest/presets/default-esm",
-    "globals": {
-      "ts-jest": {
-        "useESM": true
-      }
-    },
     "moduleNameMapper": {
       "^(\\.{1,2}/.*)\\.js$": "$1"
     },
     "testRegex": "./src/.*\\.test\\.ts$",
     "rootDir": "./src"
   },
   "dependencies": {
-    "tslib": "^2.4.0"
+    "debug": "^4.3.4",
+    "tsc-watch": "^6.0.0",
+    "tslib": "^2.5.0",
+    "viem": "^0.1.15",
+    "ws": "^8.13.0"
   },
   "devDependencies": {
     "@aztec/eslint-config": "workspace:^",
-    "@jest/globals": "^29.4.3",
-    "@rushstack/eslint-patch": "^1.1.4",
-    "@types/jest": "^29.4.0",
-    "@types/node": "^18.7.23",
-    "jest": "^28.1.3",
-    "ts-jest": "^28.0.7",
+    "@jest/globals": "^29.5.0",
+    "@rushstack/eslint-patch": "^1.2.0",
+    "@types/debug": "^4.1.7",
+    "@types/jest": "^29.4.1",
+    "@types/node": "^18.15.2",
+    "@types/ws": "^8.5.4",
+    "jest": "^29.5.0",
+    "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1",
     "typescript": "^4.9.5"
   }

yarn-project/archiver/src/abis/rollup.ts

@@ -0,0 +1,14 @@
+export const rollupAbi = [
+  {
+    name: 'L2BlockProcessed',
+    type: 'event',
+    inputs: [{ type: 'uint256', name: 'blockNum', indexed: true }],
+  },
+  {
+    inputs: [],
+    name: 'nextBlockNum',
+    outputs: [{ name: '', type: 'uint256' }],
+    stateMutability: 'view',
+    type: 'function',
+  },
+] as const;

yarn-project/archiver/src/abis/yeeter.ts

@@ -0,0 +1,5 @@
+import { parseAbiItem } from 'viem';
+
+export const yeeterAbi = [
+  parseAbiItem('event Yeet(uint256 indexed blockNum, address indexed sender, bytes blabber)'),
+] as const;

yarn-project/archiver/src/archiver.test.ts

@@ -0,0 +1,59 @@
+import { jest } from '@jest/globals';
+import { getAddress, PublicClient } from 'viem';
+import { Archiver } from './archiver.js';
+
+jest.mock('viem');
+
+describe('Archiver', () => {
+  const rollupAddress = getAddress('0x0000000000000000000000000000000000000000');
+  const yeeterAddress = getAddress('0x0000000000000000000000000000000000000000');
+  let publicClient: PublicClient;
+
+  beforeEach(() => {
+    publicClient = {
+      readContract: jest.fn().mockReturnValue(3n),
+      createEventFilter: jest.fn(),
+      getFilterLogs: jest.fn().mockReturnValue([
+        {
+          args: {
+            blockNum: 0n,
+          },
+        },
+        {
+          args: {
+            blockNum: 1n,
+          },
+        },
+        {
+          args: {
+            blockNum: 2n,
+          },
+        },
+      ]),
+      watchEvent: jest.fn().mockReturnValue(jest.fn()),
+    } as unknown as PublicClient;
+  });
+
+  it('can start, sync and stop', async () => {
+    const archiver = new Archiver(publicClient, rollupAddress, yeeterAddress);
+    let syncStatus = await archiver.getSyncStatus();
+    let latestBlockNum = archiver.getLatestBlockNum();
+    expect(syncStatus).toStrictEqual({
+      syncedToBlock: -1,
+      latestBlock: 2,
+    });
+    expect(latestBlockNum).toBe(syncStatus.syncedToBlock);
+
+    await archiver.start();
+
+    syncStatus = await archiver.getSyncStatus();
+    latestBlockNum = archiver.getLatestBlockNum();
+    expect(syncStatus).toStrictEqual({
+      syncedToBlock: 2,
+      latestBlock: 2,
+    });
+    expect(latestBlockNum).toBe(syncStatus.syncedToBlock);
+
+    archiver.stop();
+  });
+});