Merge pull request #104 from forta-protocol/add-cli-caching

add caching to cli

cli/commands/run/index.spec.ts

@@ -6,20 +6,24 @@ describe("run", () => {
   const mockContainer = {
     resolve: jest.fn()
   } as any
+  const mockCache = {
+    save: jest.fn()
+  } as any
   const mockExit = jest.spyOn(process, 'exit').mockImplementation();
 
   const resetMocks = () => {
     mockContainer.resolve.mockReset()
+    mockCache.save.mockReset()
     mockExit.mockReset()
   }
 
   beforeAll(() => {
-    run = provideRun(mockContainer)
+    run = provideRun(mockContainer, mockCache)
   })
 
   beforeEach(() => resetMocks())
 
-  it("invokes runTransaction if tx argument is provided", async () => {
+  it("invokes runTransaction if --tx argument is provided", async () => {
     const mockCliArgs = {tx: '0x123'}
     const mockRunTransaction = jest.fn()
     mockContainer.resolve.mockReturnValueOnce(mockRunTransaction)
@@ -30,10 +34,12 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runTransaction")
     expect(mockRunTransaction).toHaveBeenCalledTimes(1)
     expect(mockRunTransaction).toHaveBeenCalledWith(mockCliArgs.tx)
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(1)
   })
 
-  it("invokes runBlock if block argument is provided", async () => {
+  it("invokes runBlock if --block argument is provided", async () => {
     const mockCliArgs = {block: '0xabc'}
     const mockRunBlock = jest.fn()
     mockContainer.resolve.mockReturnValueOnce(mockRunBlock)
@@ -44,10 +50,12 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runBlock")
     expect(mockRunBlock).toHaveBeenCalledTimes(1)
     expect(mockRunBlock).toHaveBeenCalledWith(mockCliArgs.block)
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(1)
   })
 
-  it("invokes runBlockRange if range argument is provided", async () => {
+  it("invokes runBlockRange if --range argument is provided", async () => {
     const mockCliArgs = {range: '1..2'}
     const mockRunBlockRange = jest.fn()
     mockContainer.resolve.mockReturnValueOnce(mockRunBlockRange)
@@ -58,10 +66,12 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runBlockRange")
     expect(mockRunBlockRange).toHaveBeenCalledTimes(1)
     expect(mockRunBlockRange).toHaveBeenCalledWith(mockCliArgs.range)
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(1)
   })
 
-  it("invokes runFile if file argument is provided", async () => {
+  it("invokes runFile if --file argument is provided", async () => {
     const mockCliArgs = {file: 'someFile.json'}
     const mockRunFile = jest.fn()
     mockContainer.resolve.mockReturnValueOnce(mockRunFile)
@@ -72,10 +82,12 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runFile")
     expect(mockRunFile).toHaveBeenCalledTimes(1)
     expect(mockRunFile).toHaveBeenCalledWith(mockCliArgs.file)
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(1)
   })
 
-  it("invokes runProdServer if prod argument is provided", async () => {
+  it("invokes runProdServer if --prod argument is provided", async () => {
     const mockCliArgs = {prod: true}
     const mockRunProdServer = jest.fn()
     mockContainer.resolve.mockReturnValueOnce(mockRunProdServer)
@@ -86,6 +98,8 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runProdServer")
     expect(mockRunProdServer).toHaveBeenCalledTimes(1)
     expect(mockRunProdServer).toHaveBeenCalledWith()
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(0)
   })
 
@@ -100,6 +114,8 @@ describe("run", () => {
     expect(mockContainer.resolve).toHaveBeenCalledWith("runLive")
     expect(mockRunLive).toHaveBeenCalledTimes(1)
     expect(mockRunLive).toHaveBeenCalledWith()
+    expect(mockCache.save).toHaveBeenCalledTimes(1)
+    expect(mockCache.save).toHaveBeenCalledWith(true)
     expect(mockExit).toHaveBeenCalledTimes(0)
   })
 })

cli/commands/run/index.ts

@@ -1,4 +1,5 @@
 import { AwilixContainer } from 'awilix';
+import { Cache } from 'flat-cache';
 import { CommandHandler } from '../..';
 import { assertExists } from '../../utils';
 import { RunBlock } from './run.block';
@@ -9,9 +10,11 @@ import { RunTransaction } from './run.transaction';
 import { RunProdServer } from './server';
 
 export default function provideRun(
-  container: AwilixContainer
+  container: AwilixContainer,
+  cache: Cache
 ): CommandHandler {
   assertExists(container, 'container')
+  assertExists(cache, 'cache')
 
   return async function run(cliArgs: any) {
     // we manually inject the run functions here (instead of through the provide function above) so that
@@ -36,6 +39,9 @@ export default function provideRun(
       await runLive()
     }
 
+    // persist any cached blocks/txs/traces to disk
+    cache.save(true) // true = dont prune keys not used in this run
+
     // invoke process.exit() for short-lived functions, otherwise
     // a child process (i.e. python agent process) can prevent commandline from returning
     let isShortLived = cliArgs.tx || cliArgs.block || cliArgs.range || cliArgs.file

cli/di.container.ts

@@ -7,6 +7,7 @@ import shell from 'shelljs'
 import prompts from 'prompts'
 import { jsonc } from 'jsonc'
 import axios, { AxiosRequestConfig } from 'axios'
+import flatCache from 'flat-cache'
 import provideInit from "./commands/init"
 import provideRun from "./commands/run"
 import providePublish from "./commands/publish"
@@ -72,6 +73,7 @@ export default function configureContainer(commandName: CommandName, cliArgs: an
         throw new Error(`unable to parse cli package.json: ${e.message}`)
       }
     }).singleton(),
+    cache: asFunction((fortaKeystore: string) => flatCache.load('cli-cache', fortaKeystore)).singleton(),
 
     fortaKeystore: asValue(join(os.homedir(), ".forta")),
     getFortaConfig: asFunction(provideGetFortaConfig),

cli/utils/get.block.with.transactions.spec.ts

@@ -6,52 +6,87 @@ describe("getBlockWithTransactions", () => {
   const mockEthersProvider = {
     send: jest.fn()
   } as any
+  const mockCache = {
+    getKey: jest.fn(),
+    setKey: jest.fn()
+  } as any
 
   const resetMocks = () => {
     mockEthersProvider.send.mockReset()
+    mockCache.getKey.mockReset()
+    mockCache.setKey.mockReset()
   }
 
   beforeAll(() => {
-    getBlockWithTransactions = provideGetBlockWithTransactions(mockEthersProvider)
+    getBlockWithTransactions = provideGetBlockWithTransactions(mockEthersProvider, mockCache)
   })
 
   beforeEach(() => {
     resetMocks()
   })
 
-  it("for integer block number, invokes eth_getBlockByNumber jsonrpc method and returns block", async () => {
+  it("for integer block number, returns cached block if it exists", async () => {
     const mockBlockNumber = 123
     const mockBlock = { hash: "0xabc", number: mockBlockNumber }
+    mockCache.getKey.mockReturnValueOnce(mockBlock)
+
+    const block = await getBlockWithTransactions(mockBlockNumber)
+
+    expect(block).toStrictEqual(mockBlock)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(mockBlockNumber.toString())
+    expect(mockEthersProvider.send).toHaveBeenCalledTimes(0)
+    expect(mockCache.setKey).toHaveBeenCalledTimes(0)
+  })
+
+  it("for integer block number, invokes eth_getBlockByNumber jsonrpc method and returns block", async () => {
+    const mockBlockNumber = 123
+    const mockBlock = { hash: "0xaBc", number: mockBlockNumber }
     mockEthersProvider.send.mockReturnValueOnce(mockBlock)
 
     const block = await getBlockWithTransactions(mockBlockNumber)
 
     expect(block).toStrictEqual(mockBlock)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(mockBlockNumber.toString())
     expect(mockEthersProvider.send).toHaveBeenCalledTimes(1)
     expect(mockEthersProvider.send).toHaveBeenCalledWith("eth_getBlockByNumber", [ethers.utils.hexValue(mockBlockNumber), true])
+    expect(mockCache.setKey).toHaveBeenCalledTimes(2)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.hash.toLowerCase(), mockBlock)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.number.toString(), mockBlock)
   })
 
   it("for string block number, invokes eth_getBlockByNumber jsonrpc method and returns block", async () => {
     const mockBlockNumber = "123"
-    const mockBlock = { hash: "0xabc", number: mockBlockNumber }
+    const mockBlock = { hash: "0xabC", number: mockBlockNumber }
     mockEthersProvider.send.mockReturnValueOnce(mockBlock)
 
     const block = await getBlockWithTransactions(mockBlockNumber)
 
     expect(block).toStrictEqual(mockBlock)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(mockBlockNumber.toString())
     expect(mockEthersProvider.send).toHaveBeenCalledTimes(1)
     expect(mockEthersProvider.send).toHaveBeenCalledWith("eth_getBlockByNumber", [ethers.utils.hexValue(parseInt(mockBlockNumber)), true])
+    expect(mockCache.setKey).toHaveBeenCalledTimes(2)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.hash.toLowerCase(), mockBlock)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.number.toString(), mockBlock)
   })
 
   it("for string block hash, invokes eth_getBlockByHash jsonrpc method and returns block", async () => {
-    const mockBlockNumber = "0x123"
-    const mockBlock = { hash: "0xabc", number: mockBlockNumber }
+    const mockBlockHash = "0xAbc"
+    const mockBlock = { hash: mockBlockHash, number: 123 }
     mockEthersProvider.send.mockReturnValueOnce(mockBlock)
 
-    const block = await getBlockWithTransactions(mockBlockNumber)
+    const block = await getBlockWithTransactions(mockBlockHash)
 
     expect(block).toStrictEqual(mockBlock)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(mockBlockHash.toLowerCase())
     expect(mockEthersProvider.send).toHaveBeenCalledTimes(1)
-    expect(mockEthersProvider.send).toHaveBeenCalledWith("eth_getBlockByHash", [ethers.utils.hexValue(mockBlockNumber), true])
+    expect(mockEthersProvider.send).toHaveBeenCalledWith("eth_getBlockByHash", [ethers.utils.hexValue(mockBlockHash), true])
+    expect(mockCache.setKey).toHaveBeenCalledTimes(2)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.hash.toLowerCase(), mockBlock)
+    expect(mockCache.setKey).toHaveBeenCalledWith(mockBlock.number.toString(), mockBlock)
   })
 })

cli/utils/get.block.with.transactions.ts

@@ -1,6 +1,7 @@
 import { ethers, providers } from "ethers";
 import { assertExists } from ".";
 import { Block, Transaction } from "../../sdk";
+import { Cache } from 'flat-cache'
 
 export type JsonRpcTransaction = Omit<Transaction, 'nonce' | 'data'> & {
   nonce: string,
@@ -17,11 +18,18 @@ export type JsonRpcBlock = Omit<Block, 'number' | 'timestamp' | 'transactions'>
 export type GetBlockWithTransactions = (blockHashOrNumber: string | number) => Promise<JsonRpcBlock>
 
 export default function provideGetBlockWithTransactions(
-  ethersProvider: providers.JsonRpcProvider
+  ethersProvider: providers.JsonRpcProvider,
+  cache: Cache
 ) {
   assertExists(ethersProvider, 'ethersProvider')
+  assertExists(cache, 'cache')
 
-  return async function provideGetBlockWithTransactions(blockHashOrNumber: string | number) {
+  return async function getBlockWithTransactions(blockHashOrNumber: string | number) {
+    // check the cache first
+    const cachedBlock = cache.getKey(blockHashOrNumber.toString().toLowerCase())
+    if (cachedBlock) return cachedBlock
+
+    // determine whether to call getBlockByNumber (default) or getBlockByHash based on input
     let methodName = "eth_getBlockByNumber"
     if (typeof blockHashOrNumber === "string") {
       if (!blockHashOrNumber.startsWith("0x")) {
@@ -30,9 +38,14 @@ export default function provideGetBlockWithTransactions(
         methodName = "eth_getBlockByHash"
       }
     }
-    return ethersProvider.send(
+
+    // fetch the block
+    const block = await ethersProvider.send(
       methodName,
       [ethers.utils.hexValue(blockHashOrNumber), true]
     )
+    cache.setKey(block.hash.toLowerCase(), block)
+    cache.setKey(parseInt(block.number).toString(), block)
+    return block
   }
 }

cli/utils/get.trace.data.spec.ts

@@ -1,4 +1,4 @@
-import { GetTraceData, provideGetTraceData } from "./get.trace.data"
+import { GetTraceData, provideGetTraceData, getCacheKey } from "./get.trace.data"
 
 describe("getTraceData", () => {
   let getTraceData: GetTraceData
@@ -8,23 +8,29 @@ describe("getTraceData", () => {
   const mockAxios = {
     post: jest.fn()
   } as any
+  const mockCache = {
+    getKey: jest.fn(),
+    setKey: jest.fn()
+  } as any
   const mockBlockNumber = 55
   const mockTxHash = "0x123"
 
   const resetMocks = () => {
     mockAxios.post.mockReset()
+    mockCache.getKey.mockReset()
+    mockCache.setKey.mockReset()
   }
 
   beforeEach(() => resetMocks())
 
   beforeAll(() => {
     getTraceData = provideGetTraceData(
-      mockTraceRpcUrl, mockTraceBlockMethod, mockTraceTransactionMethod, mockAxios
+      mockTraceRpcUrl, mockTraceBlockMethod, mockTraceTransactionMethod, mockAxios, mockCache
     )
   })
 
   it("returns empty array if no traceRpcUrl provided", async () => {
-    const getTraceData = provideGetTraceData("", mockTraceBlockMethod, mockTraceTransactionMethod, mockAxios)
+    const getTraceData = provideGetTraceData("", mockTraceBlockMethod, mockTraceTransactionMethod, mockAxios, mockCache)
 
     const traces = await getTraceData(mockBlockNumber)
 
@@ -41,6 +47,19 @@ describe("getTraceData", () => {
     expect(mockAxios.post).toHaveBeenCalledTimes(1)
   })
 
+  it("returns cached trace data if it exists", async () => {
+    const mockTraces = ['some block trace data']
+    mockCache.getKey.mockReturnValueOnce(mockTraces)
+
+    const traces = await getTraceData(mockBlockNumber)
+
+    expect(traces).toEqual(mockTraces)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(getCacheKey(mockBlockNumber))
+    expect(mockAxios.post).toHaveBeenCalledTimes(0)
+    expect(mockCache.setKey).toHaveBeenCalledTimes(0)
+  })
+
   it("returns block trace data when requesting block number", async () => {
     const systemTime = new Date()
     jest.useFakeTimers('modern').setSystemTime(systemTime)
@@ -50,6 +69,8 @@ describe("getTraceData", () => {
     const traces = await getTraceData(mockBlockNumber)
 
     expect(traces).toEqual(mockTraces)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(getCacheKey(mockBlockNumber))
     expect(mockAxios.post).toHaveBeenCalledTimes(1)
     expect(mockAxios.post).toHaveBeenCalledWith(mockTraceRpcUrl, {
       method: mockTraceBlockMethod,
@@ -60,6 +81,8 @@ describe("getTraceData", () => {
       headers: {
         "Content-Type": "application/json",
     }})
+    expect(mockCache.setKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.setKey).toHaveBeenCalledWith(getCacheKey(mockBlockNumber), mockTraces)
     jest.useRealTimers()
   })
 
@@ -72,6 +95,8 @@ describe("getTraceData", () => {
     const traces = await getTraceData(mockTxHash)
 
     expect(traces).toEqual(mockTraces)
+    expect(mockCache.getKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.getKey).toHaveBeenCalledWith(getCacheKey(mockTxHash))
     expect(mockAxios.post).toHaveBeenCalledTimes(1)
     expect(mockAxios.post).toHaveBeenCalledWith(mockTraceRpcUrl, {
       method: mockTraceTransactionMethod,
@@ -82,6 +107,8 @@ describe("getTraceData", () => {
       headers: {
         "Content-Type": "application/json",
     }})
+    expect(mockCache.setKey).toHaveBeenCalledTimes(1)
+    expect(mockCache.setKey).toHaveBeenCalledWith(getCacheKey(mockTxHash), mockTraces)
     jest.useRealTimers()
   })
 })