Initial Archiver

README.md

@@ -7,7 +7,7 @@ The Aztec 3 system consists of the following sub projects.
 - `aztec.js`
 - `barretenberg.js`
 - `circuit.js`
-- `data-archiver`
+- `rollup-archiver`
 - `ethereum.js`
 - `kernel-simulator`
 - `key-store`

bootstrap.sh

@@ -48,7 +48,7 @@ PROJECTS=(
   "yarn-project/aztec.js:yarn build"
   # "yarn-project/barretenberg.js:yarn build"
   # "yarn-project/circuit.js:yarn build"
-  # "yarn-project/data-archiver:yarn build"
+  # "yarn-project/rollup-archiver:yarn build"
   # "yarn-project/ethereum.js:yarn build"
   # "yarn-project/kernel-simulator:yarn build"
   "yarn-project/key-store:yarn build"

build_manifest.json

@@ -73,11 +73,11 @@
     "rebuildPatterns": ["^yarn-project/circuit.js/"],
     "dependencies": ["yarn-project-base"]
   },
-  "data-archiver": {
+  "rollup-archiver": {
     "buildDir": "yarn-project",
-    "projectDir": "yarn-project/data-archiver",
-    "dockerfile": "data-archiver/Dockerfile",
-    "rebuildPatterns": ["^yarn-project/data-archiver/"],
+    "projectDir": "yarn-project/rollup-archiver",
+    "dockerfile": "rollup-archiver/Dockerfile",
+    "rebuildPatterns": ["^yarn-project/rollup-archiver/"],
     "dependencies": ["yarn-project-base"]
   },
   "end-to-end": {

build_manifest.sh

@@ -16,7 +16,7 @@ PROJECTS=(
   # aztec.js:yarn-project
   # barretenberg.js:yarn-project
   # circuit.js:yarn-project
-  # data-archiver:yarn-project
+  # rollup-archiver:yarn-project
   # end-to-end:yarn-project
   # ethereum.js:yarn-project
   # kernel-simulator:yarn-project

yarn-project/package.json

@@ -12,7 +12,7 @@
     "aztec.js",
     "barretenberg.js",
     "circuit.js",
-    "data-archiver",
+    "rollup-archiver",
     "docs",
     "end-to-end",
     "eslint-config",

yarn-project/data-archiver/Dockerfile → yarn-project/rollup-archiver/Dockerfile

@@ -1,14 +1,14 @@
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/yarn-project-base AS builder
 
-COPY data-archiver data-archiver
-WORKDIR /usr/src/yarn-project/data-archiver
+COPY rollup-archiver rollup-archiver
+WORKDIR /usr/src/yarn-project/rollup-archiver
 RUN yarn build && yarn formatting && yarn test
 
 # Prune dev dependencies. See comment in base image.
 RUN yarn cache clean
 RUN yarn workspaces focus --production > /dev/null
 
 FROM node:18-alpine
-COPY --from=builder /usr/src/yarn-project/data-archiver /usr/src/yarn-project/data-archiver
-WORKDIR /usr/src/yarn-project/data-archiver
+COPY --from=builder /usr/src/yarn-project/rollup-archiver /usr/src/yarn-project/rollup-archiver
+WORKDIR /usr/src/yarn-project/rollup-archiver
 ENTRYPOINT ["yarn"]

yarn-project/rollup-archiver/README.md

@@ -0,0 +1 @@
+# Rollup Archiver

yarn-project/data-archiver/package.json → yarn-project/rollup-archiver/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "@aztec/data-archiver",
+  "name": "@aztec/rollup-archiver",
   "version": "0.0.0",
   "type": "module",
   "exports": "./dest/index.js",
@@ -11,34 +11,34 @@
   "scripts": {
     "build": "yarn clean && yarn formatting && tsc -b tsconfig.dest.json",
     "build:dev": "tsc -b tsconfig.dest.json --watch",
+    "start": "node ./dest",
+    "start:dev": "tsc-watch -p tsconfig.dest.json --onSuccess 'yarn start'",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "formatting": "run -T prettier --check ./src && run -T eslint --max-warnings 0 ./src",
     "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules $(yarn bin jest) --no-cache --passWithNoTests"
   },
   "jest": {
     "preset": "ts-jest/presets/default-esm",
-    "globals": {
-      "ts-jest": {
-        "useESM": true
-      }
-    },
     "moduleNameMapper": {
       "^(\\.{1,2}/.*)\\.js$": "$1"
     },
     "testRegex": "./src/.*\\.test\\.ts$",
     "rootDir": "./src"
   },
   "dependencies": {
-    "tslib": "^2.4.0"
+    "debug": "^4.3.4",
+    "tsc-watch": "^6.0.0",
+    "tslib": "^2.5.0"
   },
   "devDependencies": {
     "@aztec/eslint-config": "workspace:^",
-    "@jest/globals": "^29.4.3",
-    "@rushstack/eslint-patch": "^1.1.4",
+    "@jest/globals": "^29.5.0",
+    "@rushstack/eslint-patch": "^1.2.0",
+    "@types/debug": "^4.1.7",
     "@types/jest": "^29.4.0",
-    "@types/node": "^18.7.23",
-    "jest": "^28.1.3",
-    "ts-jest": "^28.0.7",
+    "@types/node": "^18.14.6",
+    "jest": "^29.5.0",
+    "ts-jest": "^29.0.5",
     "ts-node": "^10.9.1",
     "typescript": "^4.9.5"
   }

yarn-project/rollup-archiver/src/index.ts

@@ -0,0 +1,24 @@
+import { RollupArchiver } from './rollup_archiver.js';
+
+const {
+  ETHEREUM_HOST = 'https://mainnet.infura.io/v3/9928b52099854248b3a096be07a6b23c',
+  ROLLUP_CONTRACT_ADDRESS = '0x0000000000000000000000000000000000000000',
+} = process.env;
+
+/**
+ * A function which instantiates and starts RollupArchiver.
+ */
+function main() {
+  const rollupArchiver = new RollupArchiver(ETHEREUM_HOST, ROLLUP_CONTRACT_ADDRESS);
+
+  const shutdown = async () => {
+    await rollupArchiver.stop();
+    process.exit(0);
+  };
+  process.once('SIGINT', shutdown);
+  process.once('SIGTERM', shutdown);
+
+  rollupArchiver.start();
+}
+
+main();

yarn-project/rollup-archiver/src/movetofoundation/errors/index.ts

@@ -0,0 +1 @@
+export class InterruptError extends Error {}

yarn-project/rollup-archiver/src/movetofoundation/interruptable_sleep.ts

@@ -0,0 +1,28 @@
+import { InterruptError } from './errors/index.js';
+
+export class InterruptableSleep {
+  private interruptResolve: (shouldThrow: boolean) => void = () => {};
+  private interruptPromise = new Promise<boolean>(resolve => (this.interruptResolve = resolve));
+  private timeouts: NodeJS.Timeout[] = [];
+
+  public async sleep(ms: number) {
+    let timeout!: NodeJS.Timeout;
+    const promise = new Promise<boolean>(resolve => (timeout = setTimeout(() => resolve(false), ms)));
+    this.timeouts.push(timeout);
+    const shouldThrow = await Promise.race([promise, this.interruptPromise]);
+    clearTimeout(timeout);
+    this.timeouts.splice(this.timeouts.indexOf(timeout), 1);
+    if (shouldThrow) {
+      throw new InterruptError('Interrupted.');
+    }
+  }
+
+  public interrupt(sleepShouldThrow = false) {
+    this.interruptResolve(sleepShouldThrow);
+    this.interruptPromise = new Promise(resolve => (this.interruptResolve = resolve));
+  }
+}
+
+export function sleep(ms: number) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}

yarn-project/rollup-archiver/src/movetofoundation/log/console.ts

@@ -0,0 +1,17 @@
+export type Logger = (...args: any[]) => void;
+
+class ConsoleLogger {
+  constructor(private prefix: string, private logger: (...args: any[]) => void = console.log) {}
+
+  public log(...args: any[]) {
+    this.logger(`${this.prefix}:`, ...args);
+  }
+}
+
+export function createLogger(prefix: string): Logger {
+  if (prefix) {
+    const logger = new ConsoleLogger(prefix, console.log);
+    return (...args: any[]) => logger.log(...args);
+  }
+  return console.log;
+}

yarn-project/rollup-archiver/src/movetofoundation/log/debug.ts

@@ -0,0 +1,38 @@
+import debug from 'debug';
+
+let preLogHook: ((...args: any[]) => void) | undefined;
+let postLogHook: ((...args: any[]) => void) | undefined;
+
+function theFunctionThroughWhichAllLogsPass(logger: any, ...args: any[]) {
+  if (!debug.enabled(logger.namespace)) {
+    return;
+  }
+  if (preLogHook) {
+    preLogHook(logger.namespace, ...args);
+  }
+  logger(...args);
+  if (postLogHook) {
+    postLogHook(logger.namespace, ...args);
+  }
+}
+
+export function createDebugLogger(name: string) {
+  const logger = debug(name);
+  return (...args: any[]) => theFunctionThroughWhichAllLogsPass(logger, ...args);
+}
+
+export function setPreDebugLogHook(fn: (...args: any[]) => void) {
+  preLogHook = fn;
+}
+
+export function setPostDebugLogHook(fn: (...args: any[]) => void) {
+  postLogHook = fn;
+}
+
+export function enableLogs(str: string) {
+  debug.enable(str);
+}
+
+export function isLogEnabled(str: string) {
+  return debug.enabled(str);
+}

yarn-project/rollup-archiver/src/movetofoundation/log/index.ts

@@ -0,0 +1,3 @@
+export * from './console.js';
+export * from './debug.js';
+export * from './log_history.js';

yarn-project/rollup-archiver/src/movetofoundation/log/log_history.test.ts

@@ -0,0 +1,87 @@
+import { jest } from '@jest/globals';
+import { createDebugLogger, enableLogs } from './debug.js';
+import { LogHistory } from './log_history.js';
+
+jest.useFakeTimers();
+
+describe('log history', () => {
+  let debug: (...any: any[]) => void;
+  let logHistory: LogHistory;
+  const timestemp = new Date().toISOString();
+  const name = 'test:a';
+
+  beforeEach(() => {
+    debug = createDebugLogger(name);
+    enableLogs(name);
+    logHistory = new LogHistory();
+  });
+
+  it('keeps debug logs', () => {
+    logHistory.enable();
+    expect(logHistory.getLogs()).toEqual([]);
+    debug('0');
+    debug('1', 2);
+    debug('2', { key: ['value'] }, Buffer.alloc(2));
+    expect(logHistory.getLogs()).toEqual([
+      [timestemp, name, '0'],
+      [timestemp, name, '1', 2],
+      [timestemp, name, '2', { key: ['value'] }, Buffer.alloc(2)],
+    ]);
+  });
+
+  it('does not keep logs if not enabled', () => {
+    debug('0');
+    debug('1', 2);
+    expect(logHistory.getLogs()).toEqual([]);
+  });
+
+  it('returns last n logs', () => {
+    logHistory.enable();
+    expect(logHistory.getLogs()).toEqual([]);
+    debug('0');
+    debug('1');
+    debug('2');
+    debug('3');
+    debug('4');
+    expect(logHistory.getLogs(2)).toEqual([
+      [timestemp, name, '3'],
+      [timestemp, name, '4'],
+    ]);
+  });
+
+  it('only keeps logs with enabled namespace', () => {
+    logHistory.enable();
+    const name2 = 'test:b';
+    const debug2 = createDebugLogger(name2);
+    debug('0');
+    debug2('zero');
+    expect(logHistory.getLogs()).toEqual([[timestemp, name, '0']]);
+
+    enableLogs(`${name},${name2}`);
+    debug('1', 2);
+    debug2('one', 3);
+    expect(logHistory.getLogs()).toEqual([
+      [timestemp, name, '0'],
+      [timestemp, name, '1', 2],
+      [timestemp, name2, 'one', 3],
+    ]);
+  });
+
+  it('clears all logs', () => {
+    logHistory.enable();
+    debug('0');
+    debug('1');
+    debug('2');
+    logHistory.clear();
+    expect(logHistory.getLogs()).toEqual([]);
+  });
+
+  it('clears first n logs', () => {
+    logHistory.enable();
+    debug('0');
+    debug('1');
+    debug('2');
+    logHistory.clear(2);
+    expect(logHistory.getLogs()).toEqual([[timestemp, name, '2']]);
+  });
+});

yarn-project/rollup-archiver/src/movetofoundation/log/log_history.ts

@@ -0,0 +1,21 @@
+import { setPreDebugLogHook } from './debug.js';
+
+export class LogHistory {
+  private logs: any[][] = [];
+
+  public enable() {
+    setPreDebugLogHook((...args: any[]) => {
+      this.logs.push([new Date().toISOString(), ...args]);
+    });
+  }
+
+  public getLogs(last = 0) {
+    return last ? this.logs.slice(-last) : this.logs;
+  }
+
+  public clear(count = this.logs.length) {
+    this.logs = this.logs.slice(count);
+  }
+}
+
+export const logHistory = new LogHistory();

yarn-project/rollup-archiver/src/rollup_archiver.test.ts

@@ -0,0 +1,16 @@
+import { RollupArchiver } from './rollup_archiver.js';
+
+describe('RollupArchiver', () => {
+  const ethHost = 'http://localhost:8545/';
+  const rollupContractAddress = '0x0000000000000000000000000000000000000000';
+
+  it('can start and stop', async () => {
+    const rollupArchiver = new RollupArchiver(ethHost, rollupContractAddress);
+    rollupArchiver.start();
+    await rollupArchiver.stop();
+  });
+
+  it('successfully syncs', async () => {
+    // TODO: implement once the TS mocks are replaced with actual data
+  });
+});