Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: optimize updating pools #110

Merged
merged 23 commits into from
Feb 29, 2024
Merged

fix: optimize updating pools #110

Show file tree
Hide file tree
Changes from 20 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
870e6ba
fix: optimize updating pools
AStox Feb 9, 2024
35fc002
fix: use const
AStox Feb 12, 2024
d8bf861
feat: use multicall
AStox Feb 13, 2024
6f380e5
fix: switch to multicall3
AStox Feb 13, 2024
3b9be33
fix: multicall working for pool data
AStox Feb 13, 2024
fed79be
fix: update multicall to aggregate across all pools and execute in co…
AStox Feb 14, 2024
1d0760a
fix: use multicall for loan data
AStox Feb 14, 2024
48ef388
fix: get pool update multicall working again
AStox Feb 14, 2024
a3cd5a7
fix: loans multicall working
AStox Feb 15, 2024
7c8a95c
fix: invalid poolValuation serialize error
AStox Feb 15, 2024
f953ef1
fix: fix bug in processCalls
AStox Feb 15, 2024
decc19b
fix: filter out blocktower pools from maturity date operations
AStox Feb 15, 2024
5041c5b
fix: use maker's multicall contract and update logic to better track …
AStox Feb 16, 2024
8079564
fix: fix bug in processCalls that was overwriting results
AStox Feb 17, 2024
68096f6
fix: fix loanrate and debt fetching
AStox Feb 17, 2024
e7c5657
fix: patch bug in paginatedGetter
AStox Feb 21, 2024
fd6779c
fix: patch bug in portfolioValuation and reserve tracking
AStox Feb 21, 2024
8b46468
fix: add totalRepaid and totalBorrowed
AStox Feb 28, 2024
488eb5c
fix: merge main
AStox Feb 28, 2024
ae5a067
fix: appease the linter
AStox Feb 28, 2024
aece85e
fix: move tinlake manifest to chains-tinlake
AStox Feb 29, 2024
6185d19
fix: re-add bypass blocks
AStox Feb 29, 2024
952dbe6
fix: trim multicall ABI
AStox Feb 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
236 changes: 236 additions & 0 deletions abi/multicall.abi.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,236 @@
[
{
"inputs": [
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "aggregate",
"outputs": [
{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" },
{ "internalType": "bytes[]", "name": "returnData", "type": "bytes[]" }
],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bool", "name": "allowFailure", "type": "bool" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call3[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "aggregate3",
"outputs": [
{
"components": [
{ "internalType": "bool", "name": "success", "type": "bool" },
{ "internalType": "bytes", "name": "returnData", "type": "bytes" }
],
"internalType": "struct Multicall3.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bool", "name": "allowFailure", "type": "bool" },
{ "internalType": "uint256", "name": "value", "type": "uint256" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call3Value[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "aggregate3Value",
"outputs": [
{
"components": [
{ "internalType": "bool", "name": "success", "type": "bool" },
{ "internalType": "bytes", "name": "returnData", "type": "bytes" }
],
"internalType": "struct Multicall3.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "blockAndAggregate",
"outputs": [
{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" },
{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" },
{
"components": [
{ "internalType": "bool", "name": "success", "type": "bool" },
{ "internalType": "bytes", "name": "returnData", "type": "bytes" }
],
"internalType": "struct Multicall3.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [],
"name": "getBasefee",
"outputs": [{ "internalType": "uint256", "name": "basefee", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }],
"name": "getBlockHash",
"outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getBlockNumber",
"outputs": [{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getChainId",
"outputs": [{ "internalType": "uint256", "name": "chainid", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockCoinbase",
"outputs": [{ "internalType": "address", "name": "coinbase", "type": "address" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockDifficulty",
"outputs": [{ "internalType": "uint256", "name": "difficulty", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockGasLimit",
"outputs": [{ "internalType": "uint256", "name": "gaslimit", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getCurrentBlockTimestamp",
"outputs": [{ "internalType": "uint256", "name": "timestamp", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [{ "internalType": "address", "name": "addr", "type": "address" }],
"name": "getEthBalance",
"outputs": [{ "internalType": "uint256", "name": "balance", "type": "uint256" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [],
"name": "getLastBlockHash",
"outputs": [{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" }],
"stateMutability": "view",
"type": "function"
},
{
"inputs": [
{ "internalType": "bool", "name": "requireSuccess", "type": "bool" },
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "tryAggregate",
"outputs": [
{
"components": [
{ "internalType": "bool", "name": "success", "type": "bool" },
{ "internalType": "bytes", "name": "returnData", "type": "bytes" }
],
"internalType": "struct Multicall3.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "payable",
"type": "function"
},
{
"inputs": [
{ "internalType": "bool", "name": "requireSuccess", "type": "bool" },
{
"components": [
{ "internalType": "address", "name": "target", "type": "address" },
{ "internalType": "bytes", "name": "callData", "type": "bytes" }
],
"internalType": "struct Multicall3.Call[]",
"name": "calls",
"type": "tuple[]"
}
],
"name": "tryBlockAndAggregate",
"outputs": [
{ "internalType": "uint256", "name": "blockNumber", "type": "uint256" },
{ "internalType": "bytes32", "name": "blockHash", "type": "bytes32" },
{
"components": [
{ "internalType": "bool", "name": "success", "type": "bool" },
{ "internalType": "bytes", "name": "returnData", "type": "bytes" }
],
"internalType": "struct Multicall3.Result[]",
"name": "returnData",
"type": "tuple[]"
}
],
"stateMutability": "payable",
"type": "function"
}
]
26 changes: 22 additions & 4 deletions chains-evm/eth/centrifuge.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,30 @@ network:
chainId: '1' # Ethereum Mainnet
endpoint: 'https://mainnet.infura.io/v3/a4ba76cd4be643618572e7467a444e3a'
dictionary: 'https://gx.api.subquery.network/sq/subquery/eth-dictionary'
bypassBlocks:
- '18721040-18735450'
- '18735460-18747630'
- '18747640-18763940'
dataSources:
- kind: ethereum/Runtime
startBlock: 18721030
options:
address: '0x78E9e622A57f70F1E0Ec652A4931E4e278e58142'
- kind: ethereum/Runtime
AStox marked this conversation as resolved.
Show resolved Hide resolved
startBlock: 11063000
options:
abi: navFeed
assets:
navFeed:
file: './abi/navfeed.abi.json'
reserve:
file: './abi/reserve.abi.json'
shelf:
file: './abi/shelf.abi.json'
pile:
file: './abi/pile.abi.json'
multicall:
file: './abi/multicall.abi.json'
mapping:
file: './dist/index.js'
handlers:
- handler: handleEthBlock
kind: ethereum/BlockHandler
filter:
modulo: 7000
1 change: 1 addition & 0 deletions src/config.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export const RAY_DIGITS = 27
export const RAY = bnToBn(10).pow(bnToBn(RAY_DIGITS))
export const CPREC = (digits: number) => bnToBn(10).pow(bnToBn(digits))
export const DAIMainnetAddress = '0x6b175474e89094c44da98b954eedeac495271d0f'
export const multicallAddress = '0xeefba1e63905ef1d7acba5a8513c70307c1ce441'
export const tinlakePools = [
{
id: '0x09e43329552c9d81cf205fd5f44796fbc40c822e',
Expand Down
13 changes: 5 additions & 8 deletions src/helpers/paginatedGetter.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,22 +3,19 @@ import type { Entity } from '@subql/types-core'

type StoreArgs = Parameters<typeof store.getByField>

async function _paginatedGetter(
entity: StoreArgs[0],
field: StoreArgs[1],
value: StoreArgs[2]
): Promise<Entity[]> {
async function _paginatedGetter(entity: StoreArgs[0], field: StoreArgs[1], value: StoreArgs[2]): Promise<Entity[]> {
let results: Entity[] = []
const batch = 100
let amount = 0
let entities: Entity[]
do {
const entities: Entity[] = (await store.getByField(entity, field, value, {
entities = (await store.getByField(entity, field, value, {
offset: amount,
limit: batch,
})) as Entity[]
results = results.concat(entities)
amount += results.length
} while (results.length === batch)
amount += entities.length
} while (entities.length > 0)
return results
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If there are exactly 100 entities to query, this while loop used to enter into an infinite loop. Now, we do one extra check to make sure there are no extra entities to fetch, and if not we exit the loop.

}
export const paginatedGetter = errorHandler(_paginatedGetter)
Loading
Loading