Refactor/nullifier set

zp-relayer/package.json

@@ -12,7 +12,7 @@
     "start:direct-deposit-watcher:prod": "node direct-deposit/watcher.js",
     "deploy:local": "ts-node test/deploy.ts",
     "deploy:local:rm": "rm -rf test/STATE_DIR && ts-node test/clear.ts",
-    "test:unit": "ts-mocha -r dotenv/config --paths --timeout 1000000 test/unit-tests/*.test.ts",
+    "test:unit": "DOTENV_CONFIG_PATH=test.env ts-mocha -r dotenv/config --paths --timeout 1000000 test/unit-tests/*.test.ts",
     "test:worker": "yarn deploy:local && DOTENV_CONFIG_PATH=test.env ts-mocha --exit -r dotenv/config --paths --timeout 1000000 test/worker-tests/*.test.ts && yarn deploy:local:rm"
   },
   "dependencies": {

zp-relayer/pool.ts

@@ -14,6 +14,7 @@ import { PoolState } from './state/PoolState'
 
 import type { TxType } from 'zp-memo-parser'
 import { contractCallRetry, numToHex, toTxType, truncateHexPrefix, truncateMemoTxPrefix } from './utils/helpers'
+import { Range } from '@/utils/Range'
 import { PoolCalldataParser } from './utils/PoolCalldataParser'
 import { OUTPLUSONE } from './utils/constants'
 
@@ -126,41 +127,46 @@ class Pool {
   }
 
   async syncState(startBlock: number) {
-    logger.debug('Syncing state; starting from block %d', startBlock)
-
     const localIndex = this.state.getNextIndex()
     const localRoot = this.state.getMerkleRoot()
 
     const contractIndex = await this.getContractIndex()
     const contractRoot = await this.getContractMerkleRoot(contractIndex)
 
-    logger.debug(`LOCAL ROOT: ${localRoot}; LOCAL INDEX: ${localIndex}`)
-    logger.debug(`CONTRACT ROOT: ${contractRoot}; CONTRACT INDEX: ${contractIndex}`)
+    logger.info('State summary', {
+      localIndex,
+      localRoot,
+      contractIndex,
+      contractRoot,
+    })
+
+    // Rollback state if it is ahead of the contract
+    if (contractIndex < localIndex) {
+      logger.info('Rolling back state to index %d', contractIndex)
+      this.state.rollbackTo(contractIndex)
+    }
 
     if (contractRoot === localRoot && contractIndex === localIndex) {
       logger.info('State is ok, no need to resync')
       return
     }
 
-    const numTxs = Math.floor((contractIndex - localIndex) / OUTPLUSONE)
-    const missedIndices = Array(numTxs)
-    for (let i = 0; i < numTxs; i++) {
-      missedIndices[i] = localIndex + (i + 1) * OUTPLUSONE
-    }
+    logger.debug('Syncing state; starting from block %d', startBlock)
 
     const transactSelector = '0xaf989083'
     const directDepositSelector = '0x1dc4cb33'
 
-    const lastBlockNumber = (await this.getLastBlockToProcess()) + 1
-    let toBlock = startBlock
-    for (let fromBlock = startBlock; toBlock < lastBlockNumber; fromBlock = toBlock) {
-      toBlock = Math.min(toBlock + config.eventsProcessingBatchSize, lastBlockNumber)
+    const lastBlockNumber = await this.getLastBlockToProcess()
+    const range = new Range({
+      start: startBlock,
+      end: lastBlockNumber,
+      step: config.eventsProcessingBatchSize - 1,
+    })
+
+    for (const [fromBlock, toBlock] of range) {
       const events = await getEvents(this.PoolInstance, 'Message', {
         fromBlock,
-        toBlock: toBlock - 1,
-        filter: {
-          index: missedIndices,
-        },
+        toBlock,
       })
 
       for (let i = 0; i < events.length; i++) {
@@ -204,10 +210,6 @@ class Pool {
           const memoRaw = truncateHexPrefix(parser.getField('memo', memoSize))
 
           memo = truncateMemoTxPrefix(memoRaw, txType)
-
-          // Save nullifier in confirmed state
-          const nullifier = parser.getField('nullifier')
-          await this.state.nullifiers.add([web3.utils.hexToNumberString(nullifier)])
         } else {
           throw new Error(`Unknown transaction type: ${input}`)
         }

zp-relayer/state/PoolState.ts

@@ -126,20 +126,19 @@ export class PoolState {
     this.addTx(commitIndex * OUTPLUSONE, Buffer.from(txData, 'hex'))
   }
 
-  rollbackTo(otherState: PoolState) {
+  rollbackTo(index: number) {
     const stateNextIndex = this.tree.getNextIndex()
-    const otherStateNextIndex = otherState.tree.getNextIndex()
 
-    // Index of other state should be less than index of current state
-    if (!(otherStateNextIndex < stateNextIndex)) {
+    // `index` should be less than index of current state
+    if (index >= stateNextIndex) {
       return
     }
 
     // Rollback merkle tree
-    this.tree.rollback(otherStateNextIndex)
+    this.tree.rollback(index)
 
     // Clear txs
-    for (let i = otherStateNextIndex; i < stateNextIndex; i += OUTPLUSONE) {
+    for (let i = index; i < stateNextIndex; i += OUTPLUSONE) {
       this.txs.delete(i)
     }
   }

zp-relayer/test/unit-tests/Range.test.ts

@@ -0,0 +1,31 @@
+import { expect } from 'chai'
+import { Range } from '../../utils/Range'
+
+describe('Range', () => {
+  it('correctly iterates bounded range', () => {
+    // Positive step
+    expect(Array.from(new Range({ start: 0, end: 10, step: 2 }))).eql([
+      [0, 2],
+      [3, 5],
+      [6, 8],
+      [9, 10],
+    ])
+
+    expect(Array.from(new Range({ start: 0, end: 9, step: 2 }))).eql([
+      [0, 2],
+      [3, 5],
+      [6, 8],
+      [9, 9],
+    ])
+
+    expect(Array.from(new Range({ start: 10, end: 9, step: 1 }))).eql([])
+
+    // Negative step
+    expect(Array.from(new Range({ start: 5, end: 1, step: -2 }))).eql([
+      [5, 3],
+      [2, 1],
+    ])
+
+    expect(Array.from(new Range({ start: 0, end: 5, step: -2 }))).eql([])
+  })
+})

zp-relayer/test/unit-tests/validateTx.test.ts

@@ -4,7 +4,7 @@ import { checkDeadline } from '../../validation/tx/validateTx'
 
 describe('Validation', () => {
   it('correctly checks deadline', () => {
-    // curent time + 10 sec
+    // current time + 10 sec
     const signedDeadline = toBN(Math.floor(Date.now() / 1000) + 10)
 
     expect(checkDeadline(signedDeadline, 7)).to.be.null

zp-relayer/utils/Range.ts

@@ -0,0 +1,45 @@
+type BoundedRange = [number, number]
+
+interface RangeParams {
+  start: number
+  end: number
+  step: number
+}
+
+export class Range implements Iterable<BoundedRange> {
+  constructor(private readonly params: RangeParams) {}
+  [Symbol.iterator](): Iterator<BoundedRange, any, undefined> {
+    return new RangeIterator(this.params)
+  }
+}
+
+class RangeIterator implements Iterator<BoundedRange> {
+  private order: number
+  private curRangeStart: number
+  constructor(private readonly params: RangeParams) {
+    this.order = params.step > 0 ? 1 : -1
+    this.curRangeStart = params.start
+  }
+  next(): IteratorResult<BoundedRange> {
+    if ((this.curRangeStart - this.params.end) * this.order > 0) {
+      return {
+        value: undefined,
+        done: true,
+      }
+    }
+
+    let curRangeEnd = this.curRangeStart + this.params.step
+    // Update range end we reach out of bounds
+    if ((this.params.end - curRangeEnd) * this.order <= 0) {
+      curRangeEnd = this.params.end
+    }
+
+    const res = {
+      value: [this.curRangeStart, curRangeEnd] as BoundedRange,
+      done: false,
+    }
+
+    this.curRangeStart = curRangeEnd + this.order
+    return res
+  }
+}

zp-relayer/utils/web3.ts

@@ -6,7 +6,7 @@ export async function getNonce(web3: Web3, address: string) {
   try {
     logger.debug('Getting transaction count', { address })
     const transactionCount = await web3.eth.getTransactionCount(address)
-    logger.debug('Transaction count obtained', { address, transactionCount})
+    logger.debug('Transaction count obtained', { address, transactionCount })
     return transactionCount
   } catch (e) {
     if (e instanceof Error) logger.error(e.message)

zp-relayer/validation/tx/validateTx.ts

@@ -39,10 +39,22 @@ export function checkProof(txProof: Proof, verify: (p: SnarkProof, i: Array<stri
   return null
 }
 
-export async function checkNullifier(nullifier: string, nullifierSet: NullifierSet) {
+export async function checkNullifier(nullifier: string, nullifierSet: NullifierSet, contractFallback?: Contract) {
   const inSet = await nullifierSet.isInSet(nullifier)
-  if (inSet === 0) return null
-  return new TxValidationError(`Doublespend detected in ${nullifierSet.name}`)
+  if (inSet === 1) {
+    return new TxValidationError(`Doublespend detected in ${nullifierSet.name}`)
+  }
+
+  if (contractFallback) {
+    const inContract = await contractCallRetry(contractFallback, 'nullifiers', [nullifier])
+    if (!toBN(inContract).eq(ZERO)) {
+      logger.info('Saving nullifier to redis...', {nullifier})
+      await nullifierSet.add([nullifier])
+      return new TxValidationError(`Doublespend detected in fallback contract`)
+    }
+  }
+
+  return null
 }
 
 export function checkTransferIndex(contractPoolIndex: BN, transferIndex: BN) {
@@ -226,7 +238,8 @@ export async function validateTx(
 
   await checkAssertion(() => checkPoolId(delta.poolId, pool.poolId))
   await checkAssertion(() => checkRoot(delta.transferIndex, root, pool.optimisticState))
-  await checkAssertion(() => checkNullifier(nullifier, pool.state.nullifiers))
+  await checkAssertion(() => checkNullifier(nullifier, pool.state.nullifiers, pool.PoolInstance))
+  // For optimistic nullifiers we don't need to check the contract fallback
   await checkAssertion(() => checkNullifier(nullifier, pool.optimisticState.nullifiers))
   await checkAssertion(() => checkTransferIndex(toBN(pool.optimisticState.getNextIndex()), delta.transferIndex))
   await checkAssertion(() => checkFee(fee))

zp-relayer/workers/sentTxWorker.ts

@@ -32,7 +32,7 @@ async function checkMarked(redis: Redis, id: string) {
 
 async function clearOptimisticState() {
   logger.info('Rollback optimistic state...')
-  pool.optimisticState.rollbackTo(pool.state)
+  pool.optimisticState.rollbackTo(pool.state.getNextIndex())
   logger.info('Clearing optimistic nullifiers...')
   await pool.optimisticState.nullifiers.clear()