From 870e6baea11a237189e6df0178a1093edf286834 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Fri, 9 Feb 2024 00:34:55 -0500 Subject: [PATCH 01/22] fix: optimize updating pools --- src/mappings/handlers/ethHandlers.ts | 53 +++++++++++++++------------- 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 3a885be8..0d7312d6 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -28,7 +28,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) // update pool states - for (const tinlakePool of tinlakePools) { + const poolUpdatePromises = tinlakePools.map(async (tinlakePool) => { let pool if (block.number >= tinlakePool.startBlock) { pool = await PoolService.getOrSeed(tinlakePool.id) @@ -67,7 +67,9 @@ async function _handleEthBlock(block: EthereumBlock): Promise { ) } } - } + }) + + await Promise.all(poolUpdatePromises) // Take snapshots await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') @@ -80,19 +82,16 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: string, navFeed: string) { + logger.info(`Updating loans for pool ${poolId}`) let existingLoans = await LoanService.getByPoolId(poolId) - logger.info(`Existing loans for pool ${poolId}: ${existingLoans.length}`) let loanIndex = existingLoans.length || 1 const contractLoans = [] // eslint-disable-next-line while (true) { - logger.info(`Checking loan ${loanIndex} for pool ${poolId}`) const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) - logger.info(`after shelfContract ${shelfContract}`) let response try { response = await shelfContract.token(loanIndex) - logger.info(`after response ${response}`) } catch (e) { logger.info(`Error ${e}`) break @@ -105,7 +104,9 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: contractLoans.push(loanIndex) loanIndex++ } + logger.info(`loans for pool ${poolId}: ${contractLoans.length}`) const newLoans = contractLoans.filter((loanIndex) => !existingLoans.includes(loanIndex)) + // create new loans for (const loanIndex of newLoans) { const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) @@ -116,30 +117,34 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: loan.actualMaturityDate = new Date(Number(maturityDate) * 1000) loan.save() } + logger.info(`New loans for pool ${poolId}: ${newLoans.length}`) // update all loans existingLoans = await LoanService.getByPoolId(poolId) for (const loan of existingLoans) { - const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) - const loanIndex = loan.id.split('-')[1] - const nftLocked = await shelfContract.nftLocked(loanIndex) - if (!nftLocked) { - loan.isActive = false - loan.status = LoanStatus.CLOSED - loan.save() - } - const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) - const prevDebt = loan.outstandingDebt - const debt = await pileContract.debt(loanIndex) - loan.outstandingDebt = debt.toBigInt() - const rateGroup = await pileContract.loanRates(loanIndex) - const rates = await pileContract.rates(rateGroup) - loan.interestRatePerSec = rates.ratePerSecond.toBigInt() + if (loan.status !== LoanStatus.CLOSED) { + const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) + const loanIndex = loan.id.split('-')[1] + const nftLocked = await shelfContract.nftLocked(loanIndex) + if (!nftLocked) { + loan.isActive = false + loan.status = LoanStatus.CLOSED + loan.save() + } + const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) + const prevDebt = loan.outstandingDebt + const debt = await pileContract.debt(loanIndex) + loan.outstandingDebt = debt.toBigInt() + const rateGroup = await pileContract.loanRates(loanIndex) + const rates = await pileContract.rates(rateGroup) + loan.interestRatePerSec = rates.ratePerSecond.toBigInt() - if (prevDebt > loan.outstandingDebt) { - loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + if (prevDebt > loan.outstandingDebt) { + loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + } + logger.info(`Updating loan ${loan.id} for pool ${poolId}`) + loan.save() } - loan.save() } } From 35fc002bc1167ebc702491b27b561950fe6ad669 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Sun, 11 Feb 2024 21:52:50 -0500 Subject: [PATCH 02/22] fix: use const --- src/mappings/handlers/ethHandlers.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 0d7312d6..37d1d482 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -29,13 +29,11 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // update pool states const poolUpdatePromises = tinlakePools.map(async (tinlakePool) => { - let pool if (block.number >= tinlakePool.startBlock) { - pool = await PoolService.getOrSeed(tinlakePool.id) + const pool = await PoolService.getOrSeed(tinlakePool.id) if (block.number >= tinlakePool.startBlock && pool.totalReserve == null) { pool.totalReserve = BigInt(0) pool.portfolioValuation = BigInt(0) - pool.currency pool.isActive = false pool.currencyId = currency.id await pool.save() From d8bf8616a0300a5bc3757a236c92e2916363c9f5 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Tue, 13 Feb 2024 00:41:45 -0500 Subject: [PATCH 03/22] feat: use multicall --- abi/multicall.abi.json | 23 +++++ chains-evm/eth/centrifuge.yaml | 4 +- src/config.ts | 1 + src/mappings/handlers/ethHandlers.ts | 142 +++++++++++++++++++-------- 4 files changed, 129 insertions(+), 41 deletions(-) create mode 100644 abi/multicall.abi.json diff --git a/abi/multicall.abi.json b/abi/multicall.abi.json new file mode 100644 index 00000000..f911baf7 --- /dev/null +++ b/abi/multicall.abi.json @@ -0,0 +1,23 @@ +[ + { + "constant": false, + "inputs": [ + { + "components": [ + { "name": "target", "type": "address" }, + { "name": "callData", "type": "bytes" } + ], + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate", + "outputs": [ + { "name": "blockNumber", "type": "uint256" }, + { "name": "returnData", "type": "bytes[]" } + ], + "payable": false, + "stateMutability": "nonpayable", + "type": "function" + } +] diff --git a/chains-evm/eth/centrifuge.yaml b/chains-evm/eth/centrifuge.yaml index e972428e..2a4547c3 100644 --- a/chains-evm/eth/centrifuge.yaml +++ b/chains-evm/eth/centrifuge.yaml @@ -12,7 +12,7 @@ dataSources: options: address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142' - kind: ethereum/Runtime - startBlock: 11063000 + startBlock: 16242300 options: abi: navFeed assets: @@ -24,6 +24,8 @@ dataSources: file: './abi/shelf.abi.json' pile: file: './abi/pile.abi.json' + multicall: + file: './abi/multicall.abi.json' mapping: file: './dist/index.js' handlers: diff --git a/src/config.ts b/src/config.ts index a4939caf..482fd004 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,6 +7,7 @@ export const RAY_DIGITS = 27 export const RAY = bnToBn(10).pow(bnToBn(RAY_DIGITS)) export const CPREC = (digits: number) => bnToBn(10).pow(bnToBn(digits)) export const DAIMainnetAddress = '0x6b175474e89094c44da98b954eedeac495271d0f' +export const multicallAddress = '0xcA11bde05977b3631167028862bE2a173976CA11' export const tinlakePools = [ { id: '0x09e43329552c9d81cf205fd5f44796fbc40c822e', diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 37d1d482..f526e040 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -1,17 +1,24 @@ import { Loan, LoanStatus } from '../../types' import { EthereumBlock } from '@subql/types-ethereum' -import { DAIMainnetAddress, tinlakePools } from '../../config' +import { DAIMainnetAddress, multicallAddress, tinlakePools } from '../../config' import { errorHandler } from '../../helpers/errorHandler' import { PoolService } from '../services/poolService' import { CurrencyService } from '../services/currencyService' import { BlockchainService } from '../services/blockchainService' -import { ShelfAbi__factory, NavfeedAbi__factory, ReserveAbi__factory, PileAbi__factory } from '../../types/contracts' +import { + ShelfAbi__factory, + NavfeedAbi__factory, + ReserveAbi__factory, + PileAbi__factory, + MulticallAbi__factory, +} from '../../types/contracts' import { Provider } from '@ethersproject/providers' import { TimekeeperService, getPeriodStart } from '../../helpers/timekeeperService' import { LoanService } from '../services/loanService' import { evmStateSnapshotter } from '../../helpers/stateSnapshot' const timekeeper = TimekeeperService.init() +const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { @@ -28,9 +35,12 @@ async function _handleEthBlock(block: EthereumBlock): Promise { logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) // update pool states + const poolUpdateCalls = [] const poolUpdatePromises = tinlakePools.map(async (tinlakePool) => { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) + + // initialize new pool if (block.number >= tinlakePool.startBlock && pool.totalReserve == null) { pool.totalReserve = BigInt(0) pool.portfolioValuation = BigInt(0) @@ -39,19 +49,51 @@ async function _handleEthBlock(block: EthereumBlock): Promise { await pool.save() logger.info(`Initializing pool ${tinlakePool.id}`) } + const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) + const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) + if (latestNavFeed) { - const navFeedContract = NavfeedAbi__factory.connect(latestNavFeed.address, api as unknown as Provider) - pool.portfolioValuation = (await navFeedContract.currentNAV()).toBigInt() - await pool.save() - logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) + poolUpdateCalls.push([ + latestNavFeed.address, + NavfeedAbi__factory.createInterface().encodeFunctionData('currentNAV'), + ]) } - const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) if (latestReserve) { - const reserveContract = ReserveAbi__factory.connect(latestReserve.address, api as unknown as Provider) - pool.totalReserve = (await reserveContract.totalBalance()).toBigInt() - await pool.save() - logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) + poolUpdateCalls.push([ + latestReserve.address, + ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), + ]) + } + + // if (latestNavFeed) { + // const navFeedContract = NavfeedAbi__factory.connect(latestNavFeed.address, api as unknown as Provider) + // pool.portfolioValuation = (await navFeedContract.currentNAV()).toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) + // } + + // if (latestReserve) { + // const reserveContract = ReserveAbi__factory.connect(latestReserve.address, api as unknown as Provider) + // pool.totalReserve = (await reserveContract.totalBalance()).toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) + // } + + if (poolUpdateCalls.length > 0) { + logger.info(`Fetching multicall data for pool ${tinlakePool.id}`) + const results = await multicall.aggregate(poolUpdateCalls) + logger.info(`Multicall results: ${results}`) + // if (latestNavFeed) { + // pool.portfolioValuation = results.data[0].toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) + // } + // if (latestReserve) { + // pool.totalReserve = results.data[1].toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) + // } } // Update loans if (latestNavFeed) { @@ -84,9 +126,12 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: let existingLoans = await LoanService.getByPoolId(poolId) let loanIndex = existingLoans.length || 1 const contractLoans = [] + const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) + const navFeedContract = NavfeedAbi__factory.connect(navFeed, api as unknown as Provider) + const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) + const multicallContract = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) // eslint-disable-next-line while (true) { - const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) let response try { response = await shelfContract.token(loanIndex) @@ -105,44 +150,61 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: logger.info(`loans for pool ${poolId}: ${contractLoans.length}`) const newLoans = contractLoans.filter((loanIndex) => !existingLoans.includes(loanIndex)) - // create new loans - for (const loanIndex of newLoans) { + const nftIdCalls = newLoans.map((loanIndex) => ({ + target: navFeedContract.address, + callData: navFeedContract.interface.encodeFunctionData('nftID', [loanIndex]), + })) + + const nftIdResponses = await multicallContract.aggregate(nftIdCalls) + const maturityDateCalls = newLoans.map((loanIndex) => { + const nftId = navFeedContract.interface.decodeFunctionResult('nftID', nftIdResponses[loanIndex]) + return { + target: navFeedContract.address, + callData: navFeedContract.interface.encodeFunctionData('maturityDate', [nftId]), + } + }) + const maturityDateResponses = await multicallContract.aggregate(maturityDateCalls) + + for (let i = 0; i < newLoans.length; i++) { + const loanIndex = newLoans[i] const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) - const navFeedContract = NavfeedAbi__factory.connect(navFeed, api as unknown as Provider) - const nftId = await navFeedContract['nftID(uint256)'](loanIndex) - const maturityDate = await navFeedContract.maturityDate(nftId) - loan.actualMaturityDate = new Date(Number(maturityDate) * 1000) + // const maturityDate = await navFeedContract.maturityDate(nftId) + loan.actualMaturityDate = new Date(Number(maturityDateResponses[i]) * 1000) loan.save() } logger.info(`New loans for pool ${poolId}: ${newLoans.length}`) // update all loans - existingLoans = await LoanService.getByPoolId(poolId) + existingLoans = (await LoanService.getByPoolId(poolId)).filter((loan) => loan.status !== LoanStatus.CLOSED) + const loanDetailsCalls = [] + existingLoans.forEach((loan) => { + const loanIndex = loan.id.split('-')[1] + loanDetailsCalls.push([shelf, ShelfAbi__factory.createInterface().encodeFunctionData('nftLocked', [loanIndex])]) + loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('debt', [loanIndex])]) + loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('loanRates', [loanIndex])]) + }) + const loanDetailsResponses = await multicallContract.aggregate(loanDetailsCalls) + for (const loan of existingLoans) { - if (loan.status !== LoanStatus.CLOSED) { - const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) - const loanIndex = loan.id.split('-')[1] - const nftLocked = await shelfContract.nftLocked(loanIndex) - if (!nftLocked) { - loan.isActive = false - loan.status = LoanStatus.CLOSED - loan.save() - } - const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) - const prevDebt = loan.outstandingDebt - const debt = await pileContract.debt(loanIndex) - loan.outstandingDebt = debt.toBigInt() - const rateGroup = await pileContract.loanRates(loanIndex) - const rates = await pileContract.rates(rateGroup) - loan.interestRatePerSec = rates.ratePerSecond.toBigInt() - - if (prevDebt > loan.outstandingDebt) { - loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt - } - logger.info(`Updating loan ${loan.id} for pool ${poolId}`) + const nftLocked = loanDetailsResponses[loanDetailsCalls[0]] + if (!nftLocked) { + loan.isActive = false + loan.status = LoanStatus.CLOSED loan.save() } + const prevDebt = loan.outstandingDebt + const debt = loanDetailsResponses[loanDetailsCalls[1]] + loan.outstandingDebt = debt.toBigInt() + const rateGroup = loanDetailsResponses[loanDetailsCalls[2]] + const rates = await pileContract.rates(rateGroup) + loan.interestRatePerSec = rates.ratePerSecond.toBigInt() + + if (prevDebt > loan.outstandingDebt) { + loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + } + logger.info(`Updating loan ${loan.id} for pool ${poolId}`) + loan.save() } } From 6f380e5ac2e09a5fdcdfdca951bb7c0d760ec198 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Tue, 13 Feb 2024 00:58:02 -0500 Subject: [PATCH 04/22] fix: switch to multicall3 --- abi/multicall.abi.json | 227 ++++++++++++++++++++++++++- src/mappings/handlers/ethHandlers.ts | 4 +- 2 files changed, 222 insertions(+), 9 deletions(-) diff --git a/abi/multicall.abi.json b/abi/multicall.abi.json index f911baf7..d4f9b8c4 100644 --- a/abi/multicall.abi.json +++ b/abi/multicall.abi.json @@ -1,23 +1,236 @@ [ { - "constant": false, "inputs": [ { "components": [ - { "name": "target", "type": "address" }, - { "name": "callData", "type": "bytes" } + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } ], + "internalType": "struct Multicall3.Call[]", "name": "calls", "type": "tuple[]" } ], "name": "aggregate", "outputs": [ - { "name": "blockNumber", "type": "uint256" }, - { "name": "returnData", "type": "bytes[]" } + { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, + { "internalType": "bytes[]", "name": "returnData", "type": "bytes[]" } ], - "payable": false, - "stateMutability": "nonpayable", + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bool", "name": "allowFailure", "type": "bool" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Call3[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate3", + "outputs": [ + { + "components": [ + { "internalType": "bool", "name": "success", "type": "bool" }, + { "internalType": "bytes", "name": "returnData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bool", "name": "allowFailure", "type": "bool" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Call3Value[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "aggregate3Value", + "outputs": [ + { + "components": [ + { "internalType": "bool", "name": "success", "type": "bool" }, + { "internalType": "bytes", "name": "returnData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { + "components": [ + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "blockAndAggregate", + "outputs": [ + { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, + { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, + { + "components": [ + { "internalType": "bool", "name": "success", "type": "bool" }, + { "internalType": "bytes", "name": "returnData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [], + "name": "getBasefee", + "outputs": [{ "internalType": "uint256", "name": "basefee", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], + "name": "getBlockHash", + "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getBlockNumber", + "outputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getChainId", + "outputs": [{ "internalType": "uint256", "name": "chainid", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockCoinbase", + "outputs": [{ "internalType": "address", "name": "coinbase", "type": "address" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockDifficulty", + "outputs": [{ "internalType": "uint256", "name": "difficulty", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockGasLimit", + "outputs": [{ "internalType": "uint256", "name": "gaslimit", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getCurrentBlockTimestamp", + "outputs": [{ "internalType": "uint256", "name": "timestamp", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [{ "internalType": "address", "name": "addr", "type": "address" }], + "name": "getEthBalance", + "outputs": [{ "internalType": "uint256", "name": "balance", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "getLastBlockHash", + "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, + { + "components": [ + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "tryAggregate", + "outputs": [ + { + "components": [ + { "internalType": "bool", "name": "success", "type": "bool" }, + { "internalType": "bytes", "name": "returnData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, + { + "components": [ + { "internalType": "address", "name": "target", "type": "address" }, + { "internalType": "bytes", "name": "callData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Call[]", + "name": "calls", + "type": "tuple[]" + } + ], + "name": "tryBlockAndAggregate", + "outputs": [ + { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, + { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, + { + "components": [ + { "internalType": "bool", "name": "success", "type": "bool" }, + { "internalType": "bytes", "name": "returnData", "type": "bytes" } + ], + "internalType": "struct Multicall3.Result[]", + "name": "returnData", + "type": "tuple[]" + } + ], + "stateMutability": "payable", "type": "function" } ] diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index f526e040..0efdf1a2 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -82,8 +82,8 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (poolUpdateCalls.length > 0) { logger.info(`Fetching multicall data for pool ${tinlakePool.id}`) - const results = await multicall.aggregate(poolUpdateCalls) - logger.info(`Multicall results: ${results}`) + const results = await multicall.callStatic.aggregate3(poolUpdateCalls) + logger.info(`!!!!!!!!!!!!!!!!!!!!!!!Multicall results: ${results}`) // if (latestNavFeed) { // pool.portfolioValuation = results.data[0].toBigInt() // await pool.save() From 3b9be337102ee1b6a7079b6a26281a2e63a5f1a1 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Tue, 13 Feb 2024 15:30:32 -0500 Subject: [PATCH 05/22] fix: multicall working for pool data --- src/mappings/handlers/ethHandlers.ts | 47 ++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 0efdf1a2..511fe664 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -18,7 +18,6 @@ import { LoanService } from '../services/loanService' import { evmStateSnapshotter } from '../../helpers/stateSnapshot' const timekeeper = TimekeeperService.init() -const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { @@ -36,7 +35,8 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // update pool states const poolUpdateCalls = [] - const poolUpdatePromises = tinlakePools.map(async (tinlakePool) => { + // const poolUpdatePromises = [tinlakePools[0]].map(async (tinlakePool) => { + for (const tinlakePool of [tinlakePools[0]]) { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -81,19 +81,34 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // } if (poolUpdateCalls.length > 0) { + const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching multicall data for pool ${tinlakePool.id}`) - const results = await multicall.callStatic.aggregate3(poolUpdateCalls) - logger.info(`!!!!!!!!!!!!!!!!!!!!!!!Multicall results: ${results}`) - // if (latestNavFeed) { - // pool.portfolioValuation = results.data[0].toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) - // } - // if (latestReserve) { - // pool.totalReserve = results.data[1].toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) - // } + const results = await multicall.callStatic.aggregate(poolUpdateCalls, { blockTag: blockNumber }) + logger.info(`Multicall results. Block Number: ${results[0]}`) + if (latestNavFeed) { + const navFeedContract = NavfeedAbi__factory.connect(latestNavFeed.address, api as unknown as Provider) + logger.info( + `Multicall results1.portfolioValuation: ${navFeedContract.interface.decodeFunctionResult( + 'currentNAV', + results[1][0] + )}` + ) + // pool.portfolioValuation = results.data[0].toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) + } + if (latestReserve) { + const reserveContract = ReserveAbi__factory.connect(latestReserve.address, api as unknown as Provider) + logger.info( + `Multicall results2.totalReserve: ${reserveContract.interface.decodeFunctionResult( + 'totalBalance', + results[1][1] + )}` + ) + // pool.totalReserve = results.data[1].toBigInt() + // await pool.save() + // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) + } } // Update loans if (latestNavFeed) { @@ -107,9 +122,9 @@ async function _handleEthBlock(block: EthereumBlock): Promise { ) } } - }) + } - await Promise.all(poolUpdatePromises) + // await Promise.all(poolUpdatePromises) // Take snapshots await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') From fed79be6a6ec6922c14e81807be8064069d67c2f Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Tue, 13 Feb 2024 20:03:04 -0500 Subject: [PATCH 06/22] fix: update multicall to aggregate across all pools and execute in configurable chunks --- src/mappings/handlers/ethHandlers.ts | 92 ++++++++++++++-------------- 1 file changed, 45 insertions(+), 47 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 511fe664..401fce8c 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -35,8 +35,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // update pool states const poolUpdateCalls = [] - // const poolUpdatePromises = [tinlakePools[0]].map(async (tinlakePool) => { - for (const tinlakePool of [tinlakePools[0]]) { + for (const tinlakePool of tinlakePools) { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -65,51 +64,44 @@ async function _handleEthBlock(block: EthereumBlock): Promise { ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), ]) } + } + } + if (poolUpdateCalls.length > 0) { + const callChunks = chunkArray(poolUpdateCalls, 30) + let callResults = [] + for (let i = 0; i < callChunks.length; i++) { + const chunk = callChunks[i] + const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) + logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${poolUpdateCalls.length}`) + const results = await multicall.callStatic.aggregate(chunk) + callResults = [...callResults, results[1]] + } - // if (latestNavFeed) { - // const navFeedContract = NavfeedAbi__factory.connect(latestNavFeed.address, api as unknown as Provider) - // pool.portfolioValuation = (await navFeedContract.currentNAV()).toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) - // } - - // if (latestReserve) { - // const reserveContract = ReserveAbi__factory.connect(latestReserve.address, api as unknown as Provider) - // pool.totalReserve = (await reserveContract.totalBalance()).toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) - // } - - if (poolUpdateCalls.length > 0) { - const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - logger.info(`Fetching multicall data for pool ${tinlakePool.id}`) - const results = await multicall.callStatic.aggregate(poolUpdateCalls, { blockTag: blockNumber }) - logger.info(`Multicall results. Block Number: ${results[0]}`) - if (latestNavFeed) { - const navFeedContract = NavfeedAbi__factory.connect(latestNavFeed.address, api as unknown as Provider) - logger.info( - `Multicall results1.portfolioValuation: ${navFeedContract.interface.decodeFunctionResult( - 'currentNAV', - results[1][0] - )}` - ) - // pool.portfolioValuation = results.data[0].toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) - } - if (latestReserve) { - const reserveContract = ReserveAbi__factory.connect(latestReserve.address, api as unknown as Provider) - logger.info( - `Multicall results2.totalReserve: ${reserveContract.interface.decodeFunctionResult( - 'totalBalance', - results[1][1] - )}` - ) - // pool.totalReserve = results.data[1].toBigInt() - // await pool.save() - // logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) - } + for (const tinlakePool of tinlakePools) { + const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) + const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) + const pool = await PoolService.getOrSeed(tinlakePool.id) + + // Update pool + if (latestNavFeed) { + const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( + 'currentNAV', + callResults[1][0] + )[0] + pool.portfolioValuation = currentNAV.toBigInt() + await pool.save() + logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) } + if (latestReserve) { + const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( + 'totalBalance', + callResults[1][1] + )[0] + pool.totalReserve = totalBalance.toBigInt() + await pool.save() + logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) + } + // Update loans if (latestNavFeed) { logger.info(`Updating loans for pool ${tinlakePool.id}`) @@ -124,8 +116,6 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } - // await Promise.all(poolUpdatePromises) - // Take snapshots await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') await evmStateSnapshotter('Loan', 'LoanSnapshot', block, 'loanId', 'isActive', true) @@ -230,3 +220,11 @@ function getLatestContract(contractArray, blockNumber) { null ) } + +function chunkArray(array: T[], chunkSize: number): T[][] { + const result = [] + for (let i = 0; i < array.length; i += chunkSize) { + result.push(array.slice(i, i + chunkSize)) + } + return result +} From 1d0760aa80e40d9aa94c1d95c15558f6236521b5 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 14 Feb 2024 15:30:40 -0500 Subject: [PATCH 07/22] fix: use multicall for loan data --- src/mappings/handlers/ethHandlers.ts | 134 +++++++++++++++++---------- 1 file changed, 84 insertions(+), 50 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 401fce8c..02bc44d0 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -16,11 +16,15 @@ import { Provider } from '@ethersproject/providers' import { TimekeeperService, getPeriodStart } from '../../helpers/timekeeperService' import { LoanService } from '../services/loanService' import { evmStateSnapshotter } from '../../helpers/stateSnapshot' +import { Multicall3 } from '../../types/contracts/MulticallAbi' const timekeeper = TimekeeperService.init() export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { + logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', tinlakePools.length, tinlakePools[0].id) + logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', DAIMainnetAddress) + logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', multicallAddress) if (chainId == '1') { const date = new Date(Number(block.timestamp) * 1000) const blockNumber = block.number @@ -35,7 +39,8 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // update pool states const poolUpdateCalls = [] - for (const tinlakePool of tinlakePools) { + logger.info('Updating pools:', tinlakePools.length) + for (const tinlakePool of [tinlakePools[0]]) { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -67,13 +72,14 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } if (poolUpdateCalls.length > 0) { - const callChunks = chunkArray(poolUpdateCalls, 30) + const callChunks = chunkArray(poolUpdateCalls, 10) let callResults = [] for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${poolUpdateCalls.length}`) const results = await multicall.callStatic.aggregate(chunk) + logger.info(`Results: ${results[1]}`) callResults = [...callResults, results[1]] } @@ -129,56 +135,34 @@ async function _handleEthBlock(block: EthereumBlock): Promise { async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: string, navFeed: string) { logger.info(`Updating loans for pool ${poolId}`) let existingLoans = await LoanService.getByPoolId(poolId) - let loanIndex = existingLoans.length || 1 - const contractLoans = [] - const shelfContract = ShelfAbi__factory.connect(shelf, api as unknown as Provider) - const navFeedContract = NavfeedAbi__factory.connect(navFeed, api as unknown as Provider) - const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) - const multicallContract = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - // eslint-disable-next-line - while (true) { - let response - try { - response = await shelfContract.token(loanIndex) - } catch (e) { - logger.info(`Error ${e}`) - break - } - if (!response || response.registry === '0x0000000000000000000000000000000000000000') { - logger.info(`No more loans for pool ${poolId}`) - // no more loans - break - } - contractLoans.push(loanIndex) - loanIndex++ + const newLoans = await getNewLoans(existingLoans, shelf) + + const nftIdCalls = [] + for (const loanIndex of newLoans) { + nftIdCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex])]) } - logger.info(`loans for pool ${poolId}: ${contractLoans.length}`) - const newLoans = contractLoans.filter((loanIndex) => !existingLoans.includes(loanIndex)) - - const nftIdCalls = newLoans.map((loanIndex) => ({ - target: navFeedContract.address, - callData: navFeedContract.interface.encodeFunctionData('nftID', [loanIndex]), - })) - - const nftIdResponses = await multicallContract.aggregate(nftIdCalls) - const maturityDateCalls = newLoans.map((loanIndex) => { - const nftId = navFeedContract.interface.decodeFunctionResult('nftID', nftIdResponses[loanIndex]) - return { - target: navFeedContract.address, - callData: navFeedContract.interface.encodeFunctionData('maturityDate', [nftId]), - } - }) - const maturityDateResponses = await multicallContract.aggregate(maturityDateCalls) + const nftIdResponses = await processCalls(nftIdCalls) + const nftIds = nftIdResponses.map((response) => + NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response[0]) + ) + + const maturityDateCalls = [] + for (const nftId of nftIds) { + maturityDateCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId])]) + } + const maturityDateResponses = await processCalls(maturityDateCalls) + const maturityDates = maturityDateResponses.map((response) => + NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response[0]) + ) + // create new loans for (let i = 0; i < newLoans.length; i++) { const loanIndex = newLoans[i] const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) - - // const maturityDate = await navFeedContract.maturityDate(nftId) - loan.actualMaturityDate = new Date(Number(maturityDateResponses[i]) * 1000) + loan.actualMaturityDate = new Date(Number(maturityDates[i]) * 1000) loan.save() } - logger.info(`New loans for pool ${poolId}: ${newLoans.length}`) + logger.info(`Creating ${newLoans.length} new loans for pool ${poolId}`) // update all loans existingLoans = (await LoanService.getByPoolId(poolId)).filter((loan) => loan.status !== LoanStatus.CLOSED) @@ -189,19 +173,30 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('debt', [loanIndex])]) loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('loanRates', [loanIndex])]) }) - const loanDetailsResponses = await multicallContract.aggregate(loanDetailsCalls) + const loanDetailsResponses = await processCalls(loanDetailsCalls) + const loanDetails = [] + for (let i = 0; i < loanDetailsResponses.length; i += 3) { + loanDetails.push({ + nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[i][0]), + debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1][0])[0], + loanRates: PileAbi__factory.createInterface().decodeFunctionResult('loanRates', loanDetailsResponses[i + 2][0]), + }) + } - for (const loan of existingLoans) { - const nftLocked = loanDetailsResponses[loanDetailsCalls[0]] + for (let i = 0; i < existingLoans.length; i++) { + const loan = existingLoans[i] + const nftLocked = loanDetails[i].nftLocked if (!nftLocked) { loan.isActive = false loan.status = LoanStatus.CLOSED loan.save() } const prevDebt = loan.outstandingDebt - const debt = loanDetailsResponses[loanDetailsCalls[1]] + const debt = loanDetails[i].debt loan.outstandingDebt = debt.toBigInt() - const rateGroup = loanDetailsResponses[loanDetailsCalls[2]] + const rateGroup = loanDetails[i].loanRates + + const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) const rates = await pileContract.rates(rateGroup) loan.interestRatePerSec = rates.ratePerSecond.toBigInt() @@ -213,6 +208,31 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: } } +async function getNewLoans(existingLoans: Loan[], shelfAddress: string) { + let loanIndex = existingLoans.length || 1 + const contractLoans = [] + const shelfContract = ShelfAbi__factory.connect(shelfAddress, api as unknown as Provider) + // eslint-disable-next-line + while (true) { + let response + try { + response = await shelfContract.token(loanIndex) + } catch (e) { + logger.info(`Error ${e}`) + break + } + if (!response || response.registry === '0x0000000000000000000000000000000000000000') { + logger.info('No more loans') + // no more loans + break + } + contractLoans.push(loanIndex) + loanIndex++ + } + logger.info(`loans for pool: ${contractLoans.length}`) + return contractLoans.filter((loanIndex) => !existingLoans.includes(loanIndex)) +} + function getLatestContract(contractArray, blockNumber) { return contractArray.reduce( (prev, current) => @@ -228,3 +248,17 @@ function chunkArray(array: T[], chunkSize: number): T[][] { } return result } + +async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Promise { + const callChunks = chunkArray(calls, chunkSize) + let callResults = [] + for (let i = 0; i < callChunks.length; i++) { + const chunk = callChunks[i] + const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) + logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) + const results = await multicall.callStatic.aggregate(chunk) + callResults = [...callResults, ...results[1]] + } + + return callResults +} From 48ef388925cbade0d4576693520da2cd509bb6cf Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 14 Feb 2024 18:14:45 -0500 Subject: [PATCH 08/22] fix: get pool update multicall working again --- src/mappings/handlers/ethHandlers.ts | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 02bc44d0..3a2c1470 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -22,9 +22,9 @@ const timekeeper = TimekeeperService.init() export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { - logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', tinlakePools.length, tinlakePools[0].id) - logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', DAIMainnetAddress) - logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', multicallAddress) + // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', tinlakePools.length, tinlakePools[0].id) + // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', DAIMainnetAddress) + // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', multicallAddress) if (chainId == '1') { const date = new Date(Number(block.timestamp) * 1000) const blockNumber = block.number @@ -78,32 +78,38 @@ async function _handleEthBlock(block: EthereumBlock): Promise { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${poolUpdateCalls.length}`) - const results = await multicall.callStatic.aggregate(chunk) - logger.info(`Results: ${results[1]}`) - callResults = [...callResults, results[1]] + const results = await multicall.callStatic.tryAggregate(false, chunk) + logger.info(`Results: ${results}`) + callResults = [...callResults, results.map((result) => result[1])] + logger.info(`Call results: ${callResults}`) + logger.info(`1 ${callResults[0]}`) + logger.info(`2 ${Object.keys(callResults[0])}`) + logger.info(`3 ${callResults[0][0]}`) } - for (const tinlakePool of tinlakePools) { + for (let i = 0; i < tinlakePools.length; i++) { + const tinlakePool = tinlakePools[i] const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) const pool = await PoolService.getOrSeed(tinlakePool.id) // Update pool if (latestNavFeed) { + logger.info('getting currentNAV') const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( 'currentNAV', - callResults[1][0] + callResults[i][0] )[0] - pool.portfolioValuation = currentNAV.toBigInt() + pool.portfolioValuation = currentNAV as unknown as bigint await pool.save() logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) } if (latestReserve) { const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( 'totalBalance', - callResults[1][1] + callResults[i][1] )[0] - pool.totalReserve = totalBalance.toBigInt() + pool.totalReserve = totalBalance as unknown as bigint await pool.save() logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) } From a3cd5a7f84bf96b2b048880a4ce0ebfccc6faeeb Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 14 Feb 2024 23:17:40 -0500 Subject: [PATCH 09/22] fix: loans multicall working --- src/mappings/handlers/ethHandlers.ts | 62 +++++++++++++++++----------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 3a2c1470..31909f9a 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -22,9 +22,6 @@ const timekeeper = TimekeeperService.init() export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { - // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', tinlakePools.length, tinlakePools[0].id) - // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', DAIMainnetAddress) - // logger.info('!!!!!!!!!!!!!!!!!!!!!!!!!!!', multicallAddress) if (chainId == '1') { const date = new Date(Number(block.timestamp) * 1000) const blockNumber = block.number @@ -72,22 +69,24 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } if (poolUpdateCalls.length > 0) { - const callChunks = chunkArray(poolUpdateCalls, 10) + logger.info('starting multicall') + // const callResults = await processCalls(poolUpdateCalls) + + const callChunks = chunkArray(poolUpdateCalls, 30) let callResults = [] for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${poolUpdateCalls.length}`) - const results = await multicall.callStatic.tryAggregate(false, chunk) - logger.info(`Results: ${results}`) + logger.info('Fetching') + const results = await multicall.callStatic.tryAggregate(true, chunk) + logger.info('fetched results') callResults = [...callResults, results.map((result) => result[1])] - logger.info(`Call results: ${callResults}`) - logger.info(`1 ${callResults[0]}`) - logger.info(`2 ${Object.keys(callResults[0])}`) - logger.info(`3 ${callResults[0][0]}`) + logger.info(`results: ${results.map((result) => result[1])}`) } - for (let i = 0; i < tinlakePools.length; i++) { + logger.info('finished multicall') + + for (let i = 0; i < [tinlakePools[0]].length; i++) { const tinlakePool = tinlakePools[i] const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) @@ -100,6 +99,8 @@ async function _handleEthBlock(block: EthereumBlock): Promise { 'currentNAV', callResults[i][0] )[0] + logger.info(`currentNAV: ${currentNAV}`) + logger.info(`currentNAV.length: ${currentNAV.length}`) pool.portfolioValuation = currentNAV as unknown as bigint await pool.save() logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) @@ -116,7 +117,6 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // Update loans if (latestNavFeed) { - logger.info(`Updating loans for pool ${tinlakePool.id}`) await updateLoans( tinlakePool.id, date, @@ -128,12 +128,16 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } + logger.info('starting snapshotting') // Take snapshots await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') await evmStateSnapshotter('Loan', 'LoanSnapshot', block, 'loanId', 'isActive', true) + logger.info('finished snapshotting') //Update tracking of period and continue + logger.info('Updating timekeeper') await (await timekeeper).update(blockPeriodStart) + logger.info('Updated timekeeper') } } } @@ -148,19 +152,23 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: nftIdCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex])]) } const nftIdResponses = await processCalls(nftIdCalls) - const nftIds = nftIdResponses.map((response) => - NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response[0]) + const nftIds = nftIdResponses[0].map( + (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] ) + logger.info(`got nftIds ${nftIds}; length: ${nftIds.length}`) + const maturityDateCalls = [] for (const nftId of nftIds) { maturityDateCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId])]) } const maturityDateResponses = await processCalls(maturityDateCalls) - const maturityDates = maturityDateResponses.map((response) => - NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response[0]) + const maturityDates = maturityDateResponses[0].map( + (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] ) + logger.info(`got maturityDates ${maturityDates}`) + // create new loans for (let i = 0; i < newLoans.length; i++) { const loanIndex = newLoans[i] @@ -181,15 +189,21 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: }) const loanDetailsResponses = await processCalls(loanDetailsCalls) const loanDetails = [] - for (let i = 0; i < loanDetailsResponses.length; i += 3) { + for (let i = 0; i < loanDetailsResponses[0].length; i += 3) { loanDetails.push({ - nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[i][0]), - debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1][0])[0], - loanRates: PileAbi__factory.createInterface().decodeFunctionResult('loanRates', loanDetailsResponses[i + 2][0]), + nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[0][i])[0], + debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[0][i + 1])[0], + loanRates: PileAbi__factory.createInterface().decodeFunctionResult( + 'loanRates', + loanDetailsResponses[0][i + 2] + )[0], }) } for (let i = 0; i < existingLoans.length; i++) { + logger.info(`nftLocked ${loanDetails[i].nftLocked}; length: ${loanDetails[i].nftLocked.length}`) + logger.info(`debt ${loanDetails[i].debt}; length: ${loanDetails[i].debt.length}`) + logger.info(`loanRates ${loanDetails[i].loanRates}; length: ${loanDetails[i].loanRates.length}`) const loan = existingLoans[i] const nftLocked = loanDetails[i].nftLocked if (!nftLocked) { @@ -262,8 +276,10 @@ async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Pro const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) - const results = await multicall.callStatic.aggregate(chunk) - callResults = [...callResults, ...results[1]] + const results = await multicall.callStatic.tryAggregate(false, chunk) + logger.info('fetched results') + callResults = [...callResults, results.map((result) => result[1])] + logger.info(`results: ${results.map((result) => result[1])}`) } return callResults From 7c8a95ccb44d26e05406fd397231c4829b5edc5e Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 14 Feb 2024 23:52:44 -0500 Subject: [PATCH 10/22] fix: invalid poolValuation serialize error --- src/mappings/handlers/ethHandlers.ts | 44 ++++------------------------ 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 31909f9a..9f48febe 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -36,8 +36,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // update pool states const poolUpdateCalls = [] - logger.info('Updating pools:', tinlakePools.length) - for (const tinlakePool of [tinlakePools[0]]) { + for (const tinlakePool of tinlakePools) { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -69,24 +68,9 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } if (poolUpdateCalls.length > 0) { - logger.info('starting multicall') - // const callResults = await processCalls(poolUpdateCalls) + const callResults = await processCalls(poolUpdateCalls) - const callChunks = chunkArray(poolUpdateCalls, 30) - let callResults = [] - for (let i = 0; i < callChunks.length; i++) { - const chunk = callChunks[i] - const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - logger.info('Fetching') - const results = await multicall.callStatic.tryAggregate(true, chunk) - logger.info('fetched results') - callResults = [...callResults, results.map((result) => result[1])] - logger.info(`results: ${results.map((result) => result[1])}`) - } - - logger.info('finished multicall') - - for (let i = 0; i < [tinlakePools[0]].length; i++) { + for (let i = 0; i < tinlakePools.length; i++) { const tinlakePool = tinlakePools[i] const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) @@ -94,14 +78,11 @@ async function _handleEthBlock(block: EthereumBlock): Promise { // Update pool if (latestNavFeed) { - logger.info('getting currentNAV') const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( 'currentNAV', callResults[i][0] )[0] - logger.info(`currentNAV: ${currentNAV}`) - logger.info(`currentNAV.length: ${currentNAV.length}`) - pool.portfolioValuation = currentNAV as unknown as bigint + pool.portfolioValuation = currentNAV.toBigInt() await pool.save() logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) } @@ -110,8 +91,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { 'totalBalance', callResults[i][1] )[0] - pool.totalReserve = totalBalance as unknown as bigint - await pool.save() + pool.totalReserve = totalBalance.toBigInt() logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) } @@ -128,16 +108,12 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } - logger.info('starting snapshotting') // Take snapshots await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') await evmStateSnapshotter('Loan', 'LoanSnapshot', block, 'loanId', 'isActive', true) - logger.info('finished snapshotting') //Update tracking of period and continue - logger.info('Updating timekeeper') await (await timekeeper).update(blockPeriodStart) - logger.info('Updated timekeeper') } } } @@ -156,8 +132,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] ) - logger.info(`got nftIds ${nftIds}; length: ${nftIds.length}`) - const maturityDateCalls = [] for (const nftId of nftIds) { maturityDateCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId])]) @@ -167,8 +141,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] ) - logger.info(`got maturityDates ${maturityDates}`) - // create new loans for (let i = 0; i < newLoans.length; i++) { const loanIndex = newLoans[i] @@ -201,9 +173,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: } for (let i = 0; i < existingLoans.length; i++) { - logger.info(`nftLocked ${loanDetails[i].nftLocked}; length: ${loanDetails[i].nftLocked.length}`) - logger.info(`debt ${loanDetails[i].debt}; length: ${loanDetails[i].debt.length}`) - logger.info(`loanRates ${loanDetails[i].loanRates}; length: ${loanDetails[i].loanRates.length}`) const loan = existingLoans[i] const nftLocked = loanDetails[i].nftLocked if (!nftLocked) { @@ -249,7 +218,6 @@ async function getNewLoans(existingLoans: Loan[], shelfAddress: string) { contractLoans.push(loanIndex) loanIndex++ } - logger.info(`loans for pool: ${contractLoans.length}`) return contractLoans.filter((loanIndex) => !existingLoans.includes(loanIndex)) } @@ -277,9 +245,7 @@ async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Pro const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) const results = await multicall.callStatic.tryAggregate(false, chunk) - logger.info('fetched results') callResults = [...callResults, results.map((result) => result[1])] - logger.info(`results: ${results.map((result) => result[1])}`) } return callResults From f953ef1675704df0fcf56699a04142e7d986f7fa Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Thu, 15 Feb 2024 14:56:01 -0500 Subject: [PATCH 11/22] fix: fix bug in processCalls --- src/mappings/handlers/ethHandlers.ts | 135 ++++++++++++++++----------- 1 file changed, 81 insertions(+), 54 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 9f48febe..9cc67784 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -17,6 +17,7 @@ import { TimekeeperService, getPeriodStart } from '../../helpers/timekeeperServi import { LoanService } from '../services/loanService' import { evmStateSnapshotter } from '../../helpers/stateSnapshot' import { Multicall3 } from '../../types/contracts/MulticallAbi' +import { BigNumber } from 'ethers' const timekeeper = TimekeeperService.init() @@ -80,7 +81,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (latestNavFeed) { const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( 'currentNAV', - callResults[i][0] + callResults[i * 2] )[0] pool.portfolioValuation = currentNAV.toBigInt() await pool.save() @@ -89,7 +90,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (latestReserve) { const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( 'totalBalance', - callResults[i][1] + callResults[i * 2 + 1] )[0] pool.totalReserve = totalBalance.toBigInt() logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) @@ -123,32 +124,51 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: let existingLoans = await LoanService.getByPoolId(poolId) const newLoans = await getNewLoans(existingLoans, shelf) + logger.info('getting nftIds') const nftIdCalls = [] for (const loanIndex of newLoans) { nftIdCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex])]) } - const nftIdResponses = await processCalls(nftIdCalls) - const nftIds = nftIdResponses[0].map( - (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] - ) + logger.info(`nftIdCalls.length: ${nftIdCalls.length}`) + if (nftIdCalls.length > 0) { + const nftIdResponses = await processCalls(nftIdCalls) + logger.info(`nftIdResponses.length: ${nftIdResponses.length}`) + logger.info(`nftIdResponses[0].length: ${nftIdResponses[0].length}`) + const nftIds = nftIdResponses.map( + (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] + ) + logger.info(`nftIds.length: ${nftIds.length}`) - const maturityDateCalls = [] - for (const nftId of nftIds) { - maturityDateCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId])]) - } - const maturityDateResponses = await processCalls(maturityDateCalls) - const maturityDates = maturityDateResponses[0].map( - (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] - ) + const maturityDateCalls = [] + for (const nftId of nftIds) { + maturityDateCalls.push([ + navFeed, + NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId]), + ]) + } + logger.info(`maturityDateCalls.length: ${maturityDateCalls.length}`) + const maturityDateResponses = await processCalls(maturityDateCalls) + logger.info(`maturityDateResponses[0].length: ${maturityDateResponses[0].length}`) + const maturityDates = maturityDateResponses.map( + (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] + ) + logger.info(`maturityDates.length: ${maturityDates.length}`) - // create new loans - for (let i = 0; i < newLoans.length; i++) { - const loanIndex = newLoans[i] - const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) - loan.actualMaturityDate = new Date(Number(maturityDates[i]) * 1000) - loan.save() + // create new loans + for (let i = 0; i < newLoans.length; i++) { + logger.info(`i: ${i}`) + logger.info(`maturityDate length: ${maturityDates.length}`) + logger.info(`newLoans length: ${newLoans.length}`) + logger.info(`maturityDate type of: ${typeof maturityDates[i]}`) + logger.info(`mat keys: ${Object.keys(maturityDates[i])}`) + logger.info(`mat values: ${Object.values(maturityDates[i])}`) + const loanIndex = newLoans[i] + const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) + loan.actualMaturityDate = new Date((maturityDates[i] as BigNumber).toNumber() * 1000) + loan.save() + } + logger.info(`Creating ${newLoans.length} new loans for pool ${poolId}`) } - logger.info(`Creating ${newLoans.length} new loans for pool ${poolId}`) // update all loans existingLoans = (await LoanService.getByPoolId(poolId)).filter((loan) => loan.status !== LoanStatus.CLOSED) @@ -159,41 +179,46 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('debt', [loanIndex])]) loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('loanRates', [loanIndex])]) }) - const loanDetailsResponses = await processCalls(loanDetailsCalls) - const loanDetails = [] - for (let i = 0; i < loanDetailsResponses[0].length; i += 3) { - loanDetails.push({ - nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[0][i])[0], - debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[0][i + 1])[0], - loanRates: PileAbi__factory.createInterface().decodeFunctionResult( - 'loanRates', - loanDetailsResponses[0][i + 2] - )[0], - }) - } - - for (let i = 0; i < existingLoans.length; i++) { - const loan = existingLoans[i] - const nftLocked = loanDetails[i].nftLocked - if (!nftLocked) { - loan.isActive = false - loan.status = LoanStatus.CLOSED - loan.save() + if (loanDetailsCalls.length > 0) { + const loanDetailsResponses = await processCalls(loanDetailsCalls) + const loanDetails = [] + for (let i = 0; i < loanDetailsResponses.length; i += 3) { + loanDetails.push({ + nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[i])[0], + debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1])[0], + loanRates: PileAbi__factory.createInterface().decodeFunctionResult('loanRates', loanDetailsResponses[i + 2])[0], + }) } - const prevDebt = loan.outstandingDebt - const debt = loanDetails[i].debt - loan.outstandingDebt = debt.toBigInt() - const rateGroup = loanDetails[i].loanRates - const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) - const rates = await pileContract.rates(rateGroup) - loan.interestRatePerSec = rates.ratePerSecond.toBigInt() + for (let i = 0; i < existingLoans.length; i++) { + const loan = existingLoans[i] + const nftLocked = loanDetails[i].nftLocked + if (!nftLocked) { + loan.isActive = false + loan.status = LoanStatus.CLOSED + loan.save() + } + const prevDebt = loan.outstandingDebt + const debt = loanDetails[i].debt + loan.outstandingDebt = debt.toBigInt() + const rateGroup = loanDetails[i].loanRates + + const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) + const rates = await pileContract.rates(rateGroup) + loan.interestRatePerSec = rates.ratePerSecond.toBigInt() - if (prevDebt > loan.outstandingDebt) { - loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + if (prevDebt * rates.ratePerSecond.toBigInt() * BigInt(86400) < loan.outstandingDebt) { + loan.status = LoanStatus.ACTIVE + loan.borrowedAmountByPeriod = loan.outstandingDebt - prevDebt * rates.ratePerSecond.toBigInt() * BigInt(86400) + } + + if (prevDebt > loan.outstandingDebt) { + loan.status = LoanStatus.ACTIVE + loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + } + logger.info(`Updating loan ${loan.id} for pool ${poolId}`) + loan.save() } - logger.info(`Updating loan ${loan.id} for pool ${poolId}`) - loan.save() } } @@ -237,15 +262,17 @@ function chunkArray(array: T[], chunkSize: number): T[][] { return result } -async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Promise { +async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Promise { const callChunks = chunkArray(calls, chunkSize) let callResults = [] + logger.info(`Processing ${calls.length} calls in ${callChunks.length} chunks`) for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) const results = await multicall.callStatic.tryAggregate(false, chunk) - callResults = [...callResults, results.map((result) => result[1])] + callResults = [...callResults, ...results.map((result) => result[1])] + logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) } return callResults From decc19b6b542253a24c072777161da56c7e121bd Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Thu, 15 Feb 2024 16:48:43 -0500 Subject: [PATCH 12/22] fix: filter out blocktower pools from maturity date operations --- src/mappings/handlers/ethHandlers.ts | 69 ++++++++++++++++++---------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 9cc67784..782fac7f 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -123,6 +123,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: logger.info(`Updating loans for pool ${poolId}`) let existingLoans = await LoanService.getByPoolId(poolId) const newLoans = await getNewLoans(existingLoans, shelf) + logger.info(`Found ${newLoans.length} new loans for pool ${poolId}`) logger.info('getting nftIds') const nftIdCalls = [] @@ -138,21 +139,37 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] ) logger.info(`nftIds.length: ${nftIds.length}`) + logger.info(`nftIds[0]: ${nftIds[0]}`) - const maturityDateCalls = [] - for (const nftId of nftIds) { - maturityDateCalls.push([ - navFeed, - NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId]), - ]) + // Ignore Blocktower pools, since their loans have no maturity dates + const isBlocktower = [ + '0x4597f91cc06687bdb74147c80c097a79358ed29b', + '0xb5c08534d1e73582fbd79e7c45694cad6a5c5ab2', + '0x90040f96ab8f291b6d43a8972806e977631affde', + '0x55d86d51ac3bcab7ab7d2124931fba106c8b60c7', + ].includes(poolId) + logger.info(`isBlocktower: ${isBlocktower}`) + + let maturityDates = [] + if (!isBlocktower) { + const maturityDateCalls = [] + for (const nftId of nftIds) { + maturityDateCalls.push([ + navFeed, + NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId]), + ]) + } + logger.info(`maturityDateCalls.length: ${maturityDateCalls.length}`) + const maturityDateResponses = await processCalls(maturityDateCalls) + // logger.info(`maturityDateResponses[0].length: ${maturityDateResponses[0].length}`) + logger.info(`maturityDateResponses.length: ${maturityDateResponses.length}`) + // logger.info(`maturityDateResponses[0]: ${maturityDateResponses[0]}`) + // logger.info(`maturityDateResponses[1]: ${maturityDateResponses[1]}`) + maturityDates = maturityDateResponses.map( + (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] + ) + logger.info(`maturityDates.length: ${maturityDates.length}`) } - logger.info(`maturityDateCalls.length: ${maturityDateCalls.length}`) - const maturityDateResponses = await processCalls(maturityDateCalls) - logger.info(`maturityDateResponses[0].length: ${maturityDateResponses[0].length}`) - const maturityDates = maturityDateResponses.map( - (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] - ) - logger.info(`maturityDates.length: ${maturityDates.length}`) // create new loans for (let i = 0; i < newLoans.length; i++) { @@ -160,11 +177,13 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: logger.info(`maturityDate length: ${maturityDates.length}`) logger.info(`newLoans length: ${newLoans.length}`) logger.info(`maturityDate type of: ${typeof maturityDates[i]}`) - logger.info(`mat keys: ${Object.keys(maturityDates[i])}`) - logger.info(`mat values: ${Object.values(maturityDates[i])}`) + // logger.info(`mat keys: ${Object.keys(maturityDates[i])}`) + // logger.info(`mat values: ${Object.values(maturityDates[i])}`) const loanIndex = newLoans[i] const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) - loan.actualMaturityDate = new Date((maturityDates[i] as BigNumber).toNumber() * 1000) + if (!isBlocktower) { + loan.actualMaturityDate = new Date((maturityDates[i] as BigNumber).toNumber() * 1000) + } loan.save() } logger.info(`Creating ${newLoans.length} new loans for pool ${poolId}`) @@ -172,6 +191,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: // update all loans existingLoans = (await LoanService.getByPoolId(poolId)).filter((loan) => loan.status !== LoanStatus.CLOSED) + logger.info(`Updating ${existingLoans.length} existing loans for pool ${poolId}`) const loanDetailsCalls = [] existingLoans.forEach((loan) => { const loanIndex = loan.id.split('-')[1] @@ -202,16 +222,10 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const debt = loanDetails[i].debt loan.outstandingDebt = debt.toBigInt() const rateGroup = loanDetails[i].loanRates - const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) const rates = await pileContract.rates(rateGroup) loan.interestRatePerSec = rates.ratePerSecond.toBigInt() - if (prevDebt * rates.ratePerSecond.toBigInt() * BigInt(86400) < loan.outstandingDebt) { - loan.status = LoanStatus.ACTIVE - loan.borrowedAmountByPeriod = loan.outstandingDebt - prevDebt * rates.ratePerSecond.toBigInt() * BigInt(86400) - } - if (prevDebt > loan.outstandingDebt) { loan.status = LoanStatus.ACTIVE loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt @@ -270,9 +284,14 @@ async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Pro const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) - const results = await multicall.callStatic.tryAggregate(false, chunk) - callResults = [...callResults, ...results.map((result) => result[1])] - logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) + let results = [] + try { + results = await multicall.callStatic.tryAggregate(true, chunk) + callResults = [...callResults, ...results.map((result) => result[1])] + logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) + } catch (e) { + logger.info(`Error fetching chunk ${i}: ${e}`) + } } return callResults From 5041c5b84aa437893ad3e1dd3e3ab5ecf0e26fe3 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Fri, 16 Feb 2024 16:46:12 -0500 Subject: [PATCH 13/22] fix: use maker's multicall contract and update logic to better track mutlicalls --- src/config.ts | 2 +- src/mappings/handlers/ethHandlers.ts | 206 +++++++++++++++++++-------- 2 files changed, 147 insertions(+), 61 deletions(-) diff --git a/src/config.ts b/src/config.ts index 482fd004..095c1df8 100644 --- a/src/config.ts +++ b/src/config.ts @@ -7,7 +7,7 @@ export const RAY_DIGITS = 27 export const RAY = bnToBn(10).pow(bnToBn(RAY_DIGITS)) export const CPREC = (digits: number) => bnToBn(10).pow(bnToBn(digits)) export const DAIMainnetAddress = '0x6b175474e89094c44da98b954eedeac495271d0f' -export const multicallAddress = '0xcA11bde05977b3631167028862bE2a173976CA11' +export const multicallAddress = '0xeefba1e63905ef1d7acba5a8513c70307c1ce441' export const tinlakePools = [ { id: '0x09e43329552c9d81cf205fd5f44796fbc40c822e', diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 782fac7f..03cdf2c2 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -20,6 +20,26 @@ import { Multicall3 } from '../../types/contracts/MulticallAbi' import { BigNumber } from 'ethers' const timekeeper = TimekeeperService.init() +// type PoolContract = { +// address: string +// startBlock?: number +// } + +// type TinlakePool = { +// id: string +// navFeed: PoolContract[] +// reserve: PoolContract[] +// shelf: PoolContract[] +// pile: PoolContract[] +// startBlock: number +// } + +type PoolMulticall = { + id: string + type: string + call: Multicall3.CallStruct + result: string +} export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { @@ -36,7 +56,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) // update pool states - const poolUpdateCalls = [] + const poolUpdateCalls: PoolMulticall[] = [] for (const tinlakePool of tinlakePools) { if (block.number >= tinlakePool.startBlock) { const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -55,24 +75,33 @@ async function _handleEthBlock(block: EthereumBlock): Promise { const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) if (latestNavFeed) { - poolUpdateCalls.push([ - latestNavFeed.address, - NavfeedAbi__factory.createInterface().encodeFunctionData('currentNAV'), - ]) + poolUpdateCalls.push({ + id: tinlakePool.id, + type: 'currentNAV', + call: { + target: latestNavFeed.address, + callData: NavfeedAbi__factory.createInterface().encodeFunctionData('currentNAV'), + }, + result: '', + }) } if (latestReserve) { - poolUpdateCalls.push([ - latestReserve.address, - ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), - ]) + poolUpdateCalls.push({ + id: tinlakePool.id, + type: 'totalBalance', + call: { + target: latestReserve.address, + callData: ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), + }, + result: '', + }) } } } if (poolUpdateCalls.length > 0) { const callResults = await processCalls(poolUpdateCalls) - - for (let i = 0; i < tinlakePools.length; i++) { - const tinlakePool = tinlakePools[i] + for (const callResult of callResults) { + const tinlakePool = tinlakePools.find((p) => p.id === callResult.id) const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) const pool = await PoolService.getOrSeed(tinlakePool.id) @@ -81,7 +110,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (latestNavFeed) { const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( 'currentNAV', - callResults[i * 2] + callResult.result )[0] pool.portfolioValuation = currentNAV.toBigInt() await pool.save() @@ -90,7 +119,7 @@ async function _handleEthBlock(block: EthereumBlock): Promise { if (latestReserve) { const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( 'totalBalance', - callResults[i * 2 + 1] + callResult.result )[0] pool.totalReserve = totalBalance.toBigInt() logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) @@ -119,6 +148,12 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } } +type NewLoanData = { + id: string + nftId: string + maturityDate?: unknown +} + async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: string, navFeed: string) { logger.info(`Updating loans for pool ${poolId}`) let existingLoans = await LoanService.getByPoolId(poolId) @@ -126,20 +161,34 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: logger.info(`Found ${newLoans.length} new loans for pool ${poolId}`) logger.info('getting nftIds') - const nftIdCalls = [] + const nftIdCalls: PoolMulticall[] = [] for (const loanIndex of newLoans) { - nftIdCalls.push([navFeed, NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex])]) + nftIdCalls.push({ + id: loanIndex, + call: { + target: navFeed, + callData: NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex]), + }, + type: 'nftId', + result: '', + }) } logger.info(`nftIdCalls.length: ${nftIdCalls.length}`) if (nftIdCalls.length > 0) { + const newLoanData: NewLoanData[] = [] const nftIdResponses = await processCalls(nftIdCalls) - logger.info(`nftIdResponses.length: ${nftIdResponses.length}`) - logger.info(`nftIdResponses[0].length: ${nftIdResponses[0].length}`) - const nftIds = nftIdResponses.map( - (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response)[0] - ) - logger.info(`nftIds.length: ${nftIds.length}`) - logger.info(`nftIds[0]: ${nftIds[0]}`) + for (const response of nftIdResponses) { + if (response.result) { + const data: NewLoanData = { + id: response.id, + nftId: NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response.result)[0], + } + newLoanData.push(data) + logger.info('pushed data') + } + } + logger.info(`newLoanData.length: ${newLoanData.length}`) + logger.info(`newLoanData[0]: ${newLoanData[0]}`) // Ignore Blocktower pools, since their loans have no maturity dates const isBlocktower = [ @@ -150,39 +199,39 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: ].includes(poolId) logger.info(`isBlocktower: ${isBlocktower}`) - let maturityDates = [] if (!isBlocktower) { - const maturityDateCalls = [] - for (const nftId of nftIds) { - maturityDateCalls.push([ - navFeed, - NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId]), - ]) + const maturityDateCalls: PoolMulticall[] = [] + for (const { id, nftId } of newLoanData) { + maturityDateCalls.push({ + id, + type: 'maturityDate', + call: { + target: navFeed, + callData: NavfeedAbi__factory.createInterface().encodeFunctionData('maturityDate', [nftId]), + }, + result: '', + }) } logger.info(`maturityDateCalls.length: ${maturityDateCalls.length}`) const maturityDateResponses = await processCalls(maturityDateCalls) - // logger.info(`maturityDateResponses[0].length: ${maturityDateResponses[0].length}`) logger.info(`maturityDateResponses.length: ${maturityDateResponses.length}`) - // logger.info(`maturityDateResponses[0]: ${maturityDateResponses[0]}`) - // logger.info(`maturityDateResponses[1]: ${maturityDateResponses[1]}`) - maturityDates = maturityDateResponses.map( - (response) => NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response)[0] - ) - logger.info(`maturityDates.length: ${maturityDates.length}`) + maturityDateResponses.map((response) => { + newLoanData.find((loan) => loan.id === response.id).maturityDate = + NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response.result)[0] + }) + // logger.info(`maturityDates.length: ${maturityDates.length}`) } // create new loans - for (let i = 0; i < newLoans.length; i++) { - logger.info(`i: ${i}`) - logger.info(`maturityDate length: ${maturityDates.length}`) - logger.info(`newLoans length: ${newLoans.length}`) - logger.info(`maturityDate type of: ${typeof maturityDates[i]}`) + for (const { id, maturityDate } of newLoanData) { + // logger.info(`maturityDate length: ${maturityDates.length}`) + // logger.info(`newLoans length: ${newLoans.length}`) + // logger.info(`maturityDate type of: ${typeof maturityDates[i]}`) // logger.info(`mat keys: ${Object.keys(maturityDates[i])}`) // logger.info(`mat values: ${Object.values(maturityDates[i])}`) - const loanIndex = newLoans[i] - const loan = new Loan(`${poolId}-${loanIndex}`, blockDate, poolId, true, LoanStatus.CREATED) + const loan = new Loan(`${poolId}-${id}`, blockDate, poolId, true, LoanStatus.CREATED) if (!isBlocktower) { - loan.actualMaturityDate = new Date((maturityDates[i] as BigNumber).toNumber() * 1000) + loan.actualMaturityDate = new Date((maturityDate as BigNumber).toNumber() * 1000) } loan.save() } @@ -195,18 +244,48 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const loanDetailsCalls = [] existingLoans.forEach((loan) => { const loanIndex = loan.id.split('-')[1] - loanDetailsCalls.push([shelf, ShelfAbi__factory.createInterface().encodeFunctionData('nftLocked', [loanIndex])]) - loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('debt', [loanIndex])]) - loanDetailsCalls.push([pile, PileAbi__factory.createInterface().encodeFunctionData('loanRates', [loanIndex])]) + loanDetailsCalls.push({ + id: loanIndex, + type: 'nftLocked', + call: { + target: shelf, + callData: ShelfAbi__factory.createInterface().encodeFunctionData('nftLocked', [loanIndex]), + }, + result: '', + }) + loanDetailsCalls.push({ + pool: tinlakePools.find((p) => p.id === poolId), + type: 'debt', + call: { + target: pile, + callData: PileAbi__factory.createInterface().encodeFunctionData('debt', [loanIndex]), + }, + result: '', + }) + loanDetailsCalls.push({ + pool: tinlakePools.find((p) => p.id === poolId), + type: 'loanRates', + call: { + target: pile, + callData: PileAbi__factory.createInterface().encodeFunctionData('loanRates', [loanIndex]), + }, + result: '', + }) }) if (loanDetailsCalls.length > 0) { const loanDetailsResponses = await processCalls(loanDetailsCalls) const loanDetails = [] for (let i = 0; i < loanDetailsResponses.length; i += 3) { loanDetails.push({ - nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult('nftLocked', loanDetailsResponses[i])[0], - debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1])[0], - loanRates: PileAbi__factory.createInterface().decodeFunctionResult('loanRates', loanDetailsResponses[i + 2])[0], + nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult( + 'nftLocked', + loanDetailsResponses[i].result + )[0], + debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1].result)[0], + loanRates: PileAbi__factory.createInterface().decodeFunctionResult( + 'loanRates', + loanDetailsResponses[i + 2].result + )[0], }) } @@ -276,23 +355,30 @@ function chunkArray(array: T[], chunkSize: number): T[][] { return result } -async function processCalls(calls: Multicall3.CallStruct[], chunkSize = 30): Promise { - const callChunks = chunkArray(calls, chunkSize) - let callResults = [] - logger.info(`Processing ${calls.length} calls in ${callChunks.length} chunks`) +async function processCalls(callsArray: PoolMulticall[], chunkSize = 30): Promise { + const callChunks = chunkArray(callsArray, chunkSize) + // let callResults = [] + logger.info(`Processing ${callsArray.length} calls in ${callChunks.length} chunks`) for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${calls.length}`) + logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${callsArray.length}`) let results = [] try { - results = await multicall.callStatic.tryAggregate(true, chunk) - callResults = [...callResults, ...results.map((result) => result[1])] - logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) + const calls = chunk.map((call) => call.call) + results = await multicall.callStatic.aggregate(calls) + logger.info(`Fetched ${results.length} results`) + logger.info(`results[1]: ${results[1]}`) + logger.info(`results[1].legth: ${results[1].length}`) + logger.info(`results[1][0]: ${results[1][0]}`) + logger.info(`results[1][0].length: ${results[1][0].length}`) + results[1].map((result, i) => (callsArray[i].result = result)) + // callResults = [...callResults, ...results.map((result) => result[1])] + // logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) } catch (e) { logger.info(`Error fetching chunk ${i}: ${e}`) } } - return callResults + return callsArray } From 8079564d3d26001a15e06de8a224df537c1b11fd Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Sat, 17 Feb 2024 12:44:53 -0500 Subject: [PATCH 14/22] fix: fix bug in processCalls that was overwriting results --- src/mappings/handlers/ethHandlers.ts | 72 +++++++++++++++++----------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 03cdf2c2..41cf7747 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -216,8 +216,12 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const maturityDateResponses = await processCalls(maturityDateCalls) logger.info(`maturityDateResponses.length: ${maturityDateResponses.length}`) maturityDateResponses.map((response) => { - newLoanData.find((loan) => loan.id === response.id).maturityDate = - NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response.result)[0] + logger.info(`response.id: ${response.id}`) + logger.info(`response.result: ${response.result}`) + if (response.result) { + newLoanData.find((loan) => loan.id === response.id).maturityDate = + NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response.result)[0] + } }) // logger.info(`maturityDates.length: ${maturityDates.length}`) } @@ -241,7 +245,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: // update all loans existingLoans = (await LoanService.getByPoolId(poolId)).filter((loan) => loan.status !== LoanStatus.CLOSED) logger.info(`Updating ${existingLoans.length} existing loans for pool ${poolId}`) - const loanDetailsCalls = [] + const loanDetailsCalls: PoolMulticall[] = [] existingLoans.forEach((loan) => { const loanIndex = loan.id.split('-')[1] loanDetailsCalls.push({ @@ -254,7 +258,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: result: '', }) loanDetailsCalls.push({ - pool: tinlakePools.find((p) => p.id === poolId), + id: loanIndex, type: 'debt', call: { target: pile, @@ -263,7 +267,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: result: '', }) loanDetailsCalls.push({ - pool: tinlakePools.find((p) => p.id === poolId), + id: loanIndex, type: 'loanRates', call: { target: pile, @@ -274,33 +278,49 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: }) if (loanDetailsCalls.length > 0) { const loanDetailsResponses = await processCalls(loanDetailsCalls) - const loanDetails = [] - for (let i = 0; i < loanDetailsResponses.length; i += 3) { - loanDetails.push({ - nftLocked: ShelfAbi__factory.createInterface().decodeFunctionResult( - 'nftLocked', - loanDetailsResponses[i].result - )[0], - debt: PileAbi__factory.createInterface().decodeFunctionResult('debt', loanDetailsResponses[i + 1].result)[0], - loanRates: PileAbi__factory.createInterface().decodeFunctionResult( - 'loanRates', - loanDetailsResponses[i + 2].result - )[0], - }) + logger.info(`loanDetailsResponses.length: ${loanDetailsResponses.length}`) + logger.info('!!!!!!!!!') + const loanDetails = {} + for (let i = 0; i < loanDetailsResponses.length; i++) { + logger.info(`loanDetailsResponses[${i}].id: ${loanDetailsResponses[i].id} `) + logger.info(`loanDetailsResponses[${i}].call.target: ${loanDetailsResponses[i].call.target} `) + logger.info(`loanDetailsResponses[${i}].type: ${loanDetailsResponses[i].type} `) + logger.info(`loanDetailsResponses[${i}].result: ${loanDetailsResponses[i].result} `) + if (loanDetailsResponses[i].result) { + if (!loanDetails[loanDetailsResponses[i].id]) { + loanDetails[loanDetailsResponses[i].id] = {} + } + if (loanDetailsResponses[i].type !== 'nftLocked') { + loanDetails[loanDetailsResponses[i].id].nftLocked = ShelfAbi__factory.createInterface().decodeFunctionResult( + 'nftLocked', + loanDetailsResponses[i].result + )[0] + } else if (loanDetailsResponses[i].type === 'debt') { + loanDetails[loanDetailsResponses[i].id].debt = PileAbi__factory.createInterface().decodeFunctionResult( + 'debt', + loanDetailsResponses[i].result + )[0] + } else if (loanDetailsResponses[i].type === 'loanRates') { + loanDetails[loanDetailsResponses[i].id].loanRates = PileAbi__factory.createInterface().decodeFunctionResult( + 'loanRates', + loanDetailsResponses[i].result + )[0] + } + } } for (let i = 0; i < existingLoans.length; i++) { const loan = existingLoans[i] - const nftLocked = loanDetails[i].nftLocked + const nftLocked = loanDetails[loan.id].nftLocked if (!nftLocked) { loan.isActive = false loan.status = LoanStatus.CLOSED loan.save() } const prevDebt = loan.outstandingDebt - const debt = loanDetails[i].debt + const debt = loanDetails[loan.id].debt loan.outstandingDebt = debt.toBigInt() - const rateGroup = loanDetails[i].loanRates + const rateGroup = loanDetails[loan.id].loanRates const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) const rates = await pileContract.rates(rateGroup) loan.interestRatePerSec = rates.ratePerSecond.toBigInt() @@ -367,14 +387,8 @@ async function processCalls(callsArray: PoolMulticall[], chunkSize = 30): Promis try { const calls = chunk.map((call) => call.call) results = await multicall.callStatic.aggregate(calls) - logger.info(`Fetched ${results.length} results`) - logger.info(`results[1]: ${results[1]}`) - logger.info(`results[1].legth: ${results[1].length}`) - logger.info(`results[1][0]: ${results[1][0]}`) - logger.info(`results[1][0].length: ${results[1][0].length}`) - results[1].map((result, i) => (callsArray[i].result = result)) - // callResults = [...callResults, ...results.map((result) => result[1])] - // logger.info(`Fetched ${callResults.length} of ${calls.length} calls`) + logger.info(`Fetched ${results[1].length} results`) + results[1].map((result, j) => (callsArray[i * chunkSize + j].result = result)) } catch (e) { logger.info(`Error fetching chunk ${i}: ${e}`) } From 68096f6fa107b1e4012c7191c6a217ee1dad2935 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Sat, 17 Feb 2024 13:34:44 -0500 Subject: [PATCH 15/22] fix: fix loanrate and debt fetching --- chains-evm/eth/centrifuge.yaml | 2 +- src/mappings/handlers/ethHandlers.ts | 29 ++++++++-------------------- 2 files changed, 9 insertions(+), 22 deletions(-) diff --git a/chains-evm/eth/centrifuge.yaml b/chains-evm/eth/centrifuge.yaml index 2a4547c3..66a6b57c 100644 --- a/chains-evm/eth/centrifuge.yaml +++ b/chains-evm/eth/centrifuge.yaml @@ -12,7 +12,7 @@ dataSources: options: address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142' - kind: ethereum/Runtime - startBlock: 16242300 + startBlock: 11063000 options: abi: navFeed assets: diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 41cf7747..156199bc 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -160,7 +160,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const newLoans = await getNewLoans(existingLoans, shelf) logger.info(`Found ${newLoans.length} new loans for pool ${poolId}`) - logger.info('getting nftIds') const nftIdCalls: PoolMulticall[] = [] for (const loanIndex of newLoans) { nftIdCalls.push({ @@ -173,7 +172,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: result: '', }) } - logger.info(`nftIdCalls.length: ${nftIdCalls.length}`) if (nftIdCalls.length > 0) { const newLoanData: NewLoanData[] = [] const nftIdResponses = await processCalls(nftIdCalls) @@ -184,11 +182,8 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: nftId: NavfeedAbi__factory.createInterface().decodeFunctionResult('nftID', response.result)[0], } newLoanData.push(data) - logger.info('pushed data') } } - logger.info(`newLoanData.length: ${newLoanData.length}`) - logger.info(`newLoanData[0]: ${newLoanData[0]}`) // Ignore Blocktower pools, since their loans have no maturity dates const isBlocktower = [ @@ -197,7 +192,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: '0x90040f96ab8f291b6d43a8972806e977631affde', '0x55d86d51ac3bcab7ab7d2124931fba106c8b60c7', ].includes(poolId) - logger.info(`isBlocktower: ${isBlocktower}`) if (!isBlocktower) { const maturityDateCalls: PoolMulticall[] = [] @@ -212,12 +206,8 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: result: '', }) } - logger.info(`maturityDateCalls.length: ${maturityDateCalls.length}`) const maturityDateResponses = await processCalls(maturityDateCalls) - logger.info(`maturityDateResponses.length: ${maturityDateResponses.length}`) maturityDateResponses.map((response) => { - logger.info(`response.id: ${response.id}`) - logger.info(`response.result: ${response.result}`) if (response.result) { newLoanData.find((loan) => loan.id === response.id).maturityDate = NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response.result)[0] @@ -278,14 +268,8 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: }) if (loanDetailsCalls.length > 0) { const loanDetailsResponses = await processCalls(loanDetailsCalls) - logger.info(`loanDetailsResponses.length: ${loanDetailsResponses.length}`) - logger.info('!!!!!!!!!') const loanDetails = {} for (let i = 0; i < loanDetailsResponses.length; i++) { - logger.info(`loanDetailsResponses[${i}].id: ${loanDetailsResponses[i].id} `) - logger.info(`loanDetailsResponses[${i}].call.target: ${loanDetailsResponses[i].call.target} `) - logger.info(`loanDetailsResponses[${i}].type: ${loanDetailsResponses[i].type} `) - logger.info(`loanDetailsResponses[${i}].result: ${loanDetailsResponses[i].result} `) if (loanDetailsResponses[i].result) { if (!loanDetails[loanDetailsResponses[i].id]) { loanDetails[loanDetailsResponses[i].id] = {} @@ -295,12 +279,14 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: 'nftLocked', loanDetailsResponses[i].result )[0] - } else if (loanDetailsResponses[i].type === 'debt') { + } + if (loanDetailsResponses[i].type === 'debt') { loanDetails[loanDetailsResponses[i].id].debt = PileAbi__factory.createInterface().decodeFunctionResult( 'debt', loanDetailsResponses[i].result )[0] - } else if (loanDetailsResponses[i].type === 'loanRates') { + } + if (loanDetailsResponses[i].type === 'loanRates') { loanDetails[loanDetailsResponses[i].id].loanRates = PileAbi__factory.createInterface().decodeFunctionResult( 'loanRates', loanDetailsResponses[i].result @@ -311,16 +297,17 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: for (let i = 0; i < existingLoans.length; i++) { const loan = existingLoans[i] - const nftLocked = loanDetails[loan.id].nftLocked + const loanIndex = loan.id.split('-')[1] + const nftLocked = loanDetails[loanIndex].nftLocked if (!nftLocked) { loan.isActive = false loan.status = LoanStatus.CLOSED loan.save() } const prevDebt = loan.outstandingDebt - const debt = loanDetails[loan.id].debt + const debt = loanDetails[loanIndex].debt loan.outstandingDebt = debt.toBigInt() - const rateGroup = loanDetails[loan.id].loanRates + const rateGroup = loanDetails[loanIndex].loanRates const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) const rates = await pileContract.rates(rateGroup) loan.interestRatePerSec = rates.ratePerSecond.toBigInt() From e7c5657a32245339f61aea809e63256f4b6cae40 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 21 Feb 2024 13:03:22 -0500 Subject: [PATCH 16/22] fix: patch bug in paginatedGetter --- src/helpers/paginatedGetter.ts | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/helpers/paginatedGetter.ts b/src/helpers/paginatedGetter.ts index 606aaa02..c4a47427 100644 --- a/src/helpers/paginatedGetter.ts +++ b/src/helpers/paginatedGetter.ts @@ -3,22 +3,20 @@ import type { Entity } from '@subql/types-core' type StoreArgs = Parameters -async function _paginatedGetter( - entity: StoreArgs[0], - field: StoreArgs[1], - value: StoreArgs[2] -): Promise { +async function _paginatedGetter(entity: StoreArgs[0], field: StoreArgs[1], value: StoreArgs[2]): Promise { let results: Entity[] = [] const batch = 100 let amount = 0 + let entities: Entity[] do { - const entities: Entity[] = (await store.getByField(entity, field, value, { + entities = (await store.getByField(entity, field, value, { offset: amount, limit: batch, })) as Entity[] + logger.info(`Got ${entities.length} entities`) results = results.concat(entities) - amount += results.length - } while (results.length === batch) + amount += entities.length + } while (entities.length > 0) return results } export const paginatedGetter = errorHandler(_paginatedGetter) From fd6779cd0192af49ff72e9ddbc3243bcfecdf9d4 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 21 Feb 2024 18:39:16 -0500 Subject: [PATCH 17/22] fix: patch bug in portfolioValuation and reserve tracking --- src/helpers/paginatedGetter.ts | 1 - src/mappings/handlers/ethHandlers.ts | 23 ++++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/helpers/paginatedGetter.ts b/src/helpers/paginatedGetter.ts index c4a47427..47472321 100644 --- a/src/helpers/paginatedGetter.ts +++ b/src/helpers/paginatedGetter.ts @@ -13,7 +13,6 @@ async function _paginatedGetter(entity: StoreArgs[0], field: StoreArgs[1], value offset: amount, limit: batch, })) as Entity[] - logger.info(`Got ${entities.length} entities`) results = results.concat(entities) amount += entities.length } while (entities.length > 0) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 156199bc..d0f57d79 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -100,14 +100,15 @@ async function _handleEthBlock(block: EthereumBlock): Promise { } if (poolUpdateCalls.length > 0) { const callResults = await processCalls(poolUpdateCalls) - for (const callResult of callResults) { + for (let i = 0; i < callResults.length; i++) { + const callResult = callResults[i] const tinlakePool = tinlakePools.find((p) => p.id === callResult.id) const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) const pool = await PoolService.getOrSeed(tinlakePool.id) // Update pool - if (latestNavFeed) { + if (callResult.type === 'currentNAV' && latestNavFeed) { const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( 'currentNAV', callResult.result @@ -116,12 +117,13 @@ async function _handleEthBlock(block: EthereumBlock): Promise { await pool.save() logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) } - if (latestReserve) { + if (callResult.type === 'totalBalance' && latestReserve) { const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( 'totalBalance', callResult.result )[0] pool.totalReserve = totalBalance.toBigInt() + await pool.save() logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) } @@ -299,13 +301,17 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const loan = existingLoans[i] const loanIndex = loan.id.split('-')[1] const nftLocked = loanDetails[loanIndex].nftLocked - if (!nftLocked) { + const prevDebt = loan.outstandingDebt + const debt = loanDetails[loanIndex].debt + if (debt > BigInt(0)) { + loan.status = LoanStatus.ACTIVE + } + // if the loan is not locked or the debt is 0 and the loan was active before, close it + if (!nftLocked || (loan.status === LoanStatus.ACTIVE && debt.toBigInt() === BigInt(0))) { loan.isActive = false loan.status = LoanStatus.CLOSED loan.save() } - const prevDebt = loan.outstandingDebt - const debt = loanDetails[loanIndex].debt loan.outstandingDebt = debt.toBigInt() const rateGroup = loanDetails[loanIndex].loanRates const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) @@ -313,7 +319,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: loan.interestRatePerSec = rates.ratePerSecond.toBigInt() if (prevDebt > loan.outstandingDebt) { - loan.status = LoanStatus.ACTIVE loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt } logger.info(`Updating loan ${loan.id} for pool ${poolId}`) @@ -364,17 +369,13 @@ function chunkArray(array: T[], chunkSize: number): T[][] { async function processCalls(callsArray: PoolMulticall[], chunkSize = 30): Promise { const callChunks = chunkArray(callsArray, chunkSize) - // let callResults = [] - logger.info(`Processing ${callsArray.length} calls in ${callChunks.length} chunks`) for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - logger.info(`Fetching ${chunk.length * i} to ${chunk.length * (i + 1)} of ${callsArray.length}`) let results = [] try { const calls = chunk.map((call) => call.call) results = await multicall.callStatic.aggregate(calls) - logger.info(`Fetched ${results[1].length} results`) results[1].map((result, j) => (callsArray[i * chunkSize + j].result = result)) } catch (e) { logger.info(`Error fetching chunk ${i}: ${e}`) From 8b4646895217ce12fe108283113c2d8083057d3c Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 28 Feb 2024 15:17:29 -0500 Subject: [PATCH 18/22] fix: add totalRepaid and totalBorrowed --- chains-evm/eth/centrifuge.yaml | 6 +----- src/mappings/handlers/ethHandlers.ts | 15 ++++++++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/chains-evm/eth/centrifuge.yaml b/chains-evm/eth/centrifuge.yaml index 66a6b57c..30154c06 100644 --- a/chains-evm/eth/centrifuge.yaml +++ b/chains-evm/eth/centrifuge.yaml @@ -2,10 +2,6 @@ network: chainId: '1' # Ethereum Mainnet endpoint: 'https://mainnet.infura.io/v3/a4ba76cd4be643618572e7467a444e3a' dictionary: 'https://gx.api.subquery.network/sq/subquery/eth-dictionary' - bypassBlocks: - - '18721040-18735450' - - '18735460-18747630' - - '18747640-18763940' dataSources: - kind: ethereum/Runtime startBlock: 18721030 @@ -32,4 +28,4 @@ dataSources: - handler: handleEthBlock kind: ethereum/BlockHandler filter: - modulo: 3600 + modulo: 7000 diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index d0f57d79..20a34ded 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -220,15 +220,15 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: // create new loans for (const { id, maturityDate } of newLoanData) { - // logger.info(`maturityDate length: ${maturityDates.length}`) - // logger.info(`newLoans length: ${newLoans.length}`) - // logger.info(`maturityDate type of: ${typeof maturityDates[i]}`) - // logger.info(`mat keys: ${Object.keys(maturityDates[i])}`) - // logger.info(`mat values: ${Object.values(maturityDates[i])}`) const loan = new Loan(`${poolId}-${id}`, blockDate, poolId, true, LoanStatus.CREATED) if (!isBlocktower) { loan.actualMaturityDate = new Date((maturityDate as BigNumber).toNumber() * 1000) } + loan.totalBorrowed = BigInt(0) + loan.totalRepaid = BigInt(0) + loan.outstandingDebt = BigInt(0) + loan.borrowedAmountByPeriod = BigInt(0) + loan.repaidAmountByPeriod = BigInt(0) loan.save() } logger.info(`Creating ${newLoans.length} new loans for pool ${poolId}`) @@ -320,6 +320,11 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: if (prevDebt > loan.outstandingDebt) { loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt + loan.totalRepaid += loan.repaidAmountByPeriod + } + if (prevDebt * (loan.interestRatePerSec / BigInt(10) ** BigInt(27)) * BigInt(86400) < loan.outstandingDebt) { + loan.borrowedAmountByPeriod = loan.outstandingDebt - prevDebt + loan.totalBorrowed += loan.borrowedAmountByPeriod } logger.info(`Updating loan ${loan.id} for pool ${poolId}`) loan.save() From ae5a0675f252b9d596c39125ce21784b50388988 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Wed, 28 Feb 2024 16:55:12 -0500 Subject: [PATCH 19/22] fix: appease the linter --- src/mappings/handlers/ethHandlers.ts | 252 +++++++++++++------------- src/mappings/services/assetService.ts | 9 +- 2 files changed, 133 insertions(+), 128 deletions(-) diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index d675748c..0ecd3b60 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -1,4 +1,4 @@ -import { AssetStatus } from '../../types' +import { AssetStatus, AssetType, AssetValuationMethod } from '../../types' import { EthereumBlock } from '@subql/types-ethereum' import { DAIMainnetAddress, multicallAddress, tinlakePools } from '../../config' import { errorHandler } from '../../helpers/errorHandler' @@ -20,19 +20,6 @@ import { Multicall3 } from '../../types/contracts/MulticallAbi' import { BigNumber } from 'ethers' const timekeeper = TimekeeperService.init() -// type PoolContract = { -// address: string -// startBlock?: number -// } - -// type TinlakePool = { -// id: string -// navFeed: PoolContract[] -// reserve: PoolContract[] -// shelf: PoolContract[] -// pile: PoolContract[] -// startBlock: number -// } type PoolMulticall = { id: string @@ -43,110 +30,108 @@ type PoolMulticall = { export const handleEthBlock = errorHandler(_handleEthBlock) async function _handleEthBlock(block: EthereumBlock): Promise { - if (chainId == '1') { - const date = new Date(Number(block.timestamp) * 1000) - const blockNumber = block.number - const newPeriod = (await timekeeper).processBlock(date) - const blockPeriodStart = getPeriodStart(date) + const date = new Date(Number(block.timestamp) * 1000) + const blockNumber = block.number + const newPeriod = (await timekeeper).processBlock(date) + const blockPeriodStart = getPeriodStart(date) - const blockchain = await BlockchainService.getOrInit('1') - const currency = await CurrencyService.getOrInitEvm(blockchain.id, DAIMainnetAddress) + const blockchain = await BlockchainService.getOrInit('1') + const currency = await CurrencyService.getOrInitEvm(blockchain.id, DAIMainnetAddress) - if (newPeriod) { - logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) + if (newPeriod) { + logger.info(`It's a new period on EVM block ${blockNumber}: ${date.toISOString()}`) - // update pool states - const poolUpdateCalls: PoolMulticall[] = [] - for (const tinlakePool of tinlakePools) { - if (block.number >= tinlakePool.startBlock) { - const pool = await PoolService.getOrSeed(tinlakePool.id) + // update pool states + const poolUpdateCalls: PoolMulticall[] = [] + for (const tinlakePool of tinlakePools) { + if (block.number >= tinlakePool.startBlock) { + const pool = await PoolService.getOrSeed(tinlakePool.id) - // initialize new pool - if (block.number >= tinlakePool.startBlock && pool.totalReserve == null) { - pool.totalReserve = BigInt(0) - pool.portfolioValuation = BigInt(0) - pool.isActive = false - pool.currencyId = currency.id - await pool.save() - logger.info(`Initializing pool ${tinlakePool.id}`) - } + // initialize new pool + if (block.number >= tinlakePool.startBlock && pool.totalReserve == null) { + pool.totalReserve = BigInt(0) + pool.portfolioValuation = BigInt(0) + pool.isActive = false + pool.currencyId = currency.id + await pool.save() + logger.info(`Initializing pool ${tinlakePool.id}`) + } - const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) - const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) + const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) + const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) - if (latestNavFeed) { - poolUpdateCalls.push({ - id: tinlakePool.id, - type: 'currentNAV', - call: { - target: latestNavFeed.address, - callData: NavfeedAbi__factory.createInterface().encodeFunctionData('currentNAV'), - }, - result: '', - }) - } - if (latestReserve) { - poolUpdateCalls.push({ - id: tinlakePool.id, - type: 'totalBalance', - call: { - target: latestReserve.address, - callData: ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), - }, - result: '', - }) - } + if (latestNavFeed) { + poolUpdateCalls.push({ + id: tinlakePool.id, + type: 'currentNAV', + call: { + target: latestNavFeed.address, + callData: NavfeedAbi__factory.createInterface().encodeFunctionData('currentNAV'), + }, + result: '', + }) + } + if (latestReserve) { + poolUpdateCalls.push({ + id: tinlakePool.id, + type: 'totalBalance', + call: { + target: latestReserve.address, + callData: ReserveAbi__factory.createInterface().encodeFunctionData('totalBalance'), + }, + result: '', + }) } } - if (poolUpdateCalls.length > 0) { - const callResults = await processCalls(poolUpdateCalls) - for (let i = 0; i < callResults.length; i++) { - const callResult = callResults[i] - const tinlakePool = tinlakePools.find((p) => p.id === callResult.id) - const latestNavFeed = getLatestContract(tinlakePool.navFeed, blockNumber) - const latestReserve = getLatestContract(tinlakePool.reserve, blockNumber) - const pool = await PoolService.getOrSeed(tinlakePool.id) + } + if (poolUpdateCalls.length > 0) { + const callResults = await processCalls(poolUpdateCalls) + for (let i = 0; i < callResults.length; i++) { + const callResult = callResults[i] + const tinlakePool = tinlakePools.find((p) => p.id === callResult.id) + const latestNavFeed = getLatestContract(tinlakePool?.navFeed, blockNumber) + const latestReserve = getLatestContract(tinlakePool?.reserve, blockNumber) + const pool = await PoolService.getOrSeed(tinlakePool?.id as string) - // Update pool - if (callResult.type === 'currentNAV' && latestNavFeed) { - const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( - 'currentNAV', - callResult.result - )[0] - pool.portfolioValuation = currentNAV.toBigInt() - await pool.save() - logger.info(`Updating pool ${tinlakePool.id} with portfolioValuation: ${pool.portfolioValuation}`) - } - if (callResult.type === 'totalBalance' && latestReserve) { - const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( - 'totalBalance', - callResult.result - )[0] - pool.totalReserve = totalBalance.toBigInt() - await pool.save() - logger.info(`Updating pool ${tinlakePool.id} with totalReserve: ${pool.totalReserve}`) - } + // Update pool + if (callResult.type === 'currentNAV' && latestNavFeed) { + const currentNAV = NavfeedAbi__factory.createInterface().decodeFunctionResult( + 'currentNAV', + callResult.result + )[0] + pool.portfolioValuation = currentNAV.toBigInt() + await pool.save() + logger.info(`Updating pool ${tinlakePool?.id} with portfolioValuation: ${pool.portfolioValuation}`) + } + if (callResult.type === 'totalBalance' && latestReserve) { + const totalBalance = ReserveAbi__factory.createInterface().decodeFunctionResult( + 'totalBalance', + callResult.result + )[0] + pool.totalReserve = totalBalance.toBigInt() + await pool.save() + logger.info(`Updating pool ${tinlakePool?.id} with totalReserve: ${pool.totalReserve}`) + } - // Update loans - if (latestNavFeed) { - await updateLoans( - tinlakePool.id, - date, - tinlakePool.shelf[0].address, - tinlakePool.pile[0].address, - latestNavFeed.address - ) - } + // Update loans + if (latestNavFeed) { + await updateLoans( + tinlakePool?.id as string, + date, + tinlakePool?.shelf[0].address as string, + tinlakePool?.pile[0].address as string, + latestNavFeed.address + ) } } + } - // Take snapshots - await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') - await evmStateSnapshotter('Loan', 'LoanSnapshot', block, 'loanId', 'isActive', true) + // Take snapshots + await evmStateSnapshotter('Pool', 'PoolSnapshot', block, 'poolId') + await evmStateSnapshotter('Loan', 'LoanSnapshot', block, 'loanId', 'isActive', true) - //Update tracking of period and continue - await (await timekeeper).update(blockPeriodStart) - } + //Update tracking of period and continue + await (await timekeeper).update(blockPeriodStart) } } @@ -159,13 +144,14 @@ type NewLoanData = { async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: string, navFeed: string) { logger.info(`Updating loans for pool ${poolId}`) let existingLoans = await AssetService.getByPoolId(poolId) - const newLoans = await getNewLoans(existingLoans, shelf) + const existingLoanIds = existingLoans?.map((loan) => parseInt(loan.id.split('-')[1])) + const newLoans = await getNewLoans(existingLoanIds as number[], shelf) logger.info(`Found ${newLoans.length} new loans for pool ${poolId}`) const nftIdCalls: PoolMulticall[] = [] for (const loanIndex of newLoans) { nftIdCalls.push({ - id: loanIndex, + id: loanIndex.toString(), call: { target: navFeed, callData: NavfeedAbi__factory.createInterface().encodeFunctionData('nftID', [loanIndex]), @@ -211,8 +197,13 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const maturityDateResponses = await processCalls(maturityDateCalls) maturityDateResponses.map((response) => { if (response.result) { - newLoanData.find((loan) => loan.id === response.id).maturityDate = - NavfeedAbi__factory.createInterface().decodeFunctionResult('maturityDate', response.result)[0] + const loan = newLoanData.find((loan) => loan.id === response.id) + if (loan) { + loan.maturityDate = NavfeedAbi__factory.createInterface().decodeFunctionResult( + 'maturityDate', + response.result + )[0] + } } }) // logger.info(`maturityDates.length: ${maturityDates.length}`) @@ -220,7 +211,15 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: // create new loans for (const { id, maturityDate } of newLoanData) { - const loan = new Loan(`${poolId}-${id}`, blockDate, poolId, true, AssetStatus.CREATED) + const loan = AssetService.init( + poolId, + id, + AssetType.Other, + AssetValuationMethod.DiscountedCashFlow, + undefined, + undefined, + blockDate + ) if (!isBlocktower) { loan.actualMaturityDate = new Date((maturityDate as BigNumber).toNumber() * 1000) } @@ -235,8 +234,8 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: } // update all loans - existingLoans = (await AssetService.getByPoolId(poolId)).filter((loan) => loan.status !== AssetStatus.CLOSED) - logger.info(`Updating ${existingLoans.length} existing loans for pool ${poolId}`) + existingLoans = (await AssetService.getByPoolId(poolId))?.filter((loan) => loan.status !== AssetStatus.CLOSED) || [] + logger.info(`Updating ${existingLoans?.length} existing loans for pool ${poolId}`) const loanDetailsCalls: PoolMulticall[] = [] existingLoans.forEach((loan) => { const loanIndex = loan.id.split('-')[1] @@ -301,7 +300,7 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: const loan = existingLoans[i] const loanIndex = loan.id.split('-')[1] const nftLocked = loanDetails[loanIndex].nftLocked - const prevDebt = loan.outstandingDebt + const prevDebt = loan.outstandingDebt || BigInt(0) const debt = loanDetails[loanIndex].debt if (debt > BigInt(0)) { loan.status = AssetStatus.ACTIVE @@ -313,18 +312,26 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: loan.save() } loan.outstandingDebt = debt.toBigInt() + const currentDebt = loan.outstandingDebt || BigInt(0) const rateGroup = loanDetails[loanIndex].loanRates const pileContract = PileAbi__factory.connect(pile, api as unknown as Provider) const rates = await pileContract.rates(rateGroup) loan.interestRatePerSec = rates.ratePerSecond.toBigInt() - if (prevDebt > loan.outstandingDebt) { - loan.repaidAmountByPeriod = prevDebt - loan.outstandingDebt - loan.totalRepaid += loan.repaidAmountByPeriod + if (prevDebt > currentDebt) { + loan.repaidAmountByPeriod = prevDebt - currentDebt + loan.totalRepaid + ? (loan.totalRepaid += loan.repaidAmountByPeriod) + : (loan.totalRepaid = loan.repaidAmountByPeriod) } - if (prevDebt * (loan.interestRatePerSec / BigInt(10) ** BigInt(27)) * BigInt(86400) < loan.outstandingDebt) { - loan.borrowedAmountByPeriod = loan.outstandingDebt - prevDebt - loan.totalBorrowed += loan.borrowedAmountByPeriod + if ( + prevDebt * (loan.interestRatePerSec / BigInt(10) ** BigInt(27)) * BigInt(86400) < + (loan.outstandingDebt || BigInt(0)) + ) { + loan.borrowedAmountByPeriod = (loan.outstandingDebt || BigInt(0)) - prevDebt + loan.totalBorrowed + ? (loan.totalBorrowed += loan.borrowedAmountByPeriod) + : (loan.totalBorrowed = loan.borrowedAmountByPeriod) } logger.info(`Updating loan ${loan.id} for pool ${poolId}`) loan.save() @@ -332,9 +339,9 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: } } -async function getNewLoans(existingLoans: Loan[], shelfAddress: string) { +async function getNewLoans(existingLoans: number[], shelfAddress: string) { let loanIndex = existingLoans.length || 1 - const contractLoans = [] + const contractLoans: number[] = [] const shelfContract = ShelfAbi__factory.connect(shelfAddress, api as unknown as Provider) // eslint-disable-next-line while (true) { @@ -365,7 +372,7 @@ function getLatestContract(contractArray, blockNumber) { } function chunkArray(array: T[], chunkSize: number): T[][] { - const result = [] + const result: T[][] = [] for (let i = 0; i < array.length; i += chunkSize) { result.push(array.slice(i, i + chunkSize)) } @@ -377,7 +384,8 @@ async function processCalls(callsArray: PoolMulticall[], chunkSize = 30): Promis for (let i = 0; i < callChunks.length; i++) { const chunk = callChunks[i] const multicall = MulticallAbi__factory.connect(multicallAddress, api as unknown as Provider) - let results = [] + // eslint-disable-next-line + let results: any[] = [] try { const calls = chunk.map((call) => call.call) results = await multicall.callStatic.aggregate(calls) diff --git a/src/mappings/services/assetService.ts b/src/mappings/services/assetService.ts index cda7c0ac..8a5276b3 100644 --- a/src/mappings/services/assetService.ts +++ b/src/mappings/services/assetService.ts @@ -11,8 +11,8 @@ export class AssetService extends Asset { assetId: string, type: AssetType, valuationMethod: AssetValuationMethod, - nftClassId: bigint, - nftItemId: bigint, + nftClassId: bigint | undefined, + nftItemId: bigint | undefined, timestamp: Date ) { logger.info(`Initialising asset ${assetId} for pool ${poolId}`) @@ -129,10 +129,7 @@ export class AssetService extends Asset { break case 'External': principal = nToBigInt( - principalObject.asExternal.quantity - .toBn() - .mul(principalObject.asExternal.settlementPrice.toBn()) - .div(WAD) + principalObject.asExternal.quantity.toBn().mul(principalObject.asExternal.settlementPrice.toBn()).div(WAD) ) break } From aece85ee693f067cc3d23e9bc70e801bdc8bd920 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Thu, 29 Feb 2024 13:25:56 -0500 Subject: [PATCH 20/22] fix: move tinlake manifest to chains-tinlake --- chains-evm/eth/centrifuge.yaml | 22 ---------------------- chains-tinlake/centrifuge.yaml | 2 ++ src/mappings/handlers/ethHandlers.ts | 1 - 3 files changed, 2 insertions(+), 23 deletions(-) diff --git a/chains-evm/eth/centrifuge.yaml b/chains-evm/eth/centrifuge.yaml index 30154c06..343f6880 100644 --- a/chains-evm/eth/centrifuge.yaml +++ b/chains-evm/eth/centrifuge.yaml @@ -7,25 +7,3 @@ dataSources: startBlock: 18721030 options: address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142' - - kind: ethereum/Runtime - startBlock: 11063000 - options: - abi: navFeed - assets: - navFeed: - file: './abi/navfeed.abi.json' - reserve: - file: './abi/reserve.abi.json' - shelf: - file: './abi/shelf.abi.json' - pile: - file: './abi/pile.abi.json' - multicall: - file: './abi/multicall.abi.json' - mapping: - file: './dist/index.js' - handlers: - - handler: handleEthBlock - kind: ethereum/BlockHandler - filter: - modulo: 7000 diff --git a/chains-tinlake/centrifuge.yaml b/chains-tinlake/centrifuge.yaml index f81dcb92..c7766c7c 100644 --- a/chains-tinlake/centrifuge.yaml +++ b/chains-tinlake/centrifuge.yaml @@ -16,6 +16,8 @@ dataSources: file: ./abi/shelf.abi.json pile: file: ./abi/pile.abi.json + multicall: + file: ./abi/multicall.abi.json mapping: file: ./dist/index.js handlers: diff --git a/src/mappings/handlers/ethHandlers.ts b/src/mappings/handlers/ethHandlers.ts index 0ecd3b60..9f9fa7a3 100644 --- a/src/mappings/handlers/ethHandlers.ts +++ b/src/mappings/handlers/ethHandlers.ts @@ -206,7 +206,6 @@ async function updateLoans(poolId: string, blockDate: Date, shelf: string, pile: } } }) - // logger.info(`maturityDates.length: ${maturityDates.length}`) } // create new loans From 6185d195652a3f441a80802691a7c97ee2cb8842 Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Thu, 29 Feb 2024 13:27:10 -0500 Subject: [PATCH 21/22] fix: re-add bypass blocks --- chains-evm/eth/centrifuge.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chains-evm/eth/centrifuge.yaml b/chains-evm/eth/centrifuge.yaml index 343f6880..b2c9adad 100644 --- a/chains-evm/eth/centrifuge.yaml +++ b/chains-evm/eth/centrifuge.yaml @@ -2,6 +2,10 @@ network: chainId: '1' # Ethereum Mainnet endpoint: 'https://mainnet.infura.io/v3/a4ba76cd4be643618572e7467a444e3a' dictionary: 'https://gx.api.subquery.network/sq/subquery/eth-dictionary' + bypassBlocks: + - '18721040-18735450' + - '18735460-18747630' + - '18747640-18763940' dataSources: - kind: ethereum/Runtime startBlock: 18721030 From 952dbe6dc815ff13a32ab2f898b0df63fef1565d Mon Sep 17 00:00:00 2001 From: Adam Stox Date: Thu, 29 Feb 2024 13:29:20 -0500 Subject: [PATCH 22/22] fix: trim multicall ABI --- abi/multicall.abi.json | 214 ----------------------------------------- 1 file changed, 214 deletions(-) diff --git a/abi/multicall.abi.json b/abi/multicall.abi.json index d4f9b8c4..64f04c60 100644 --- a/abi/multicall.abi.json +++ b/abi/multicall.abi.json @@ -18,219 +18,5 @@ ], "stateMutability": "payable", "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bool", "name": "allowFailure", "type": "bool" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call3[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "aggregate3", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bool", "name": "allowFailure", "type": "bool" }, - { "internalType": "uint256", "name": "value", "type": "uint256" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call3Value[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "aggregate3Value", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "blockAndAggregate", - "outputs": [ - { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, - { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [], - "name": "getBasefee", - "outputs": [{ "internalType": "uint256", "name": "basefee", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], - "name": "getBlockHash", - "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getBlockNumber", - "outputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getChainId", - "outputs": [{ "internalType": "uint256", "name": "chainid", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockCoinbase", - "outputs": [{ "internalType": "address", "name": "coinbase", "type": "address" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockDifficulty", - "outputs": [{ "internalType": "uint256", "name": "difficulty", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockGasLimit", - "outputs": [{ "internalType": "uint256", "name": "gaslimit", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getCurrentBlockTimestamp", - "outputs": [{ "internalType": "uint256", "name": "timestamp", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [{ "internalType": "address", "name": "addr", "type": "address" }], - "name": "getEthBalance", - "outputs": [{ "internalType": "uint256", "name": "balance", "type": "uint256" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [], - "name": "getLastBlockHash", - "outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }], - "stateMutability": "view", - "type": "function" - }, - { - "inputs": [ - { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "tryAggregate", - "outputs": [ - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" - }, - { - "inputs": [ - { "internalType": "bool", "name": "requireSuccess", "type": "bool" }, - { - "components": [ - { "internalType": "address", "name": "target", "type": "address" }, - { "internalType": "bytes", "name": "callData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Call[]", - "name": "calls", - "type": "tuple[]" - } - ], - "name": "tryBlockAndAggregate", - "outputs": [ - { "internalType": "uint256", "name": "blockNumber", "type": "uint256" }, - { "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }, - { - "components": [ - { "internalType": "bool", "name": "success", "type": "bool" }, - { "internalType": "bytes", "name": "returnData", "type": "bytes" } - ], - "internalType": "struct Multicall3.Result[]", - "name": "returnData", - "type": "tuple[]" - } - ], - "stateMutability": "payable", - "type": "function" } ]