From 65e2fe72c812821940bf1c73436dfd3800045ba7 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Thu, 7 Mar 2024 09:49:10 +0100 Subject: [PATCH 01/13] feat: setup db --- .gitignore | 2 +- drills/ethers-typechain-contract/drill.ts | 3 +- drills/xrpl-account/drill.ts | 4 +- drills/xrpl-ledgers/drill.ts | 4 +- package-lock.json | 706 +++++++++++++++--- package.json | 4 +- .../src/EthersTypechainContractIndexer.ts | 26 +- .../packages/typechain-contract/src/types.ts | 4 +- packages/ethers/src/EthersIndexer.ts | 3 +- packages/ethers/src/types.ts | 5 +- packages/vanilla/package.json | 9 +- packages/vanilla/src/EventEmitter.ts | 23 - packages/vanilla/src/Indexer.ts | 93 +-- .../vanilla/src/IndexerState.repository.ts | 78 -- packages/vanilla/src/Provider.ts | 2 +- packages/vanilla/src/db/SQLiteDB.ts | 46 ++ packages/vanilla/src/db/entities/Entity.ts | 41 + packages/vanilla/src/db/entities/LastEvent.ts | 6 + .../vanilla/src/db/entities/PendingEvent.ts | 24 + packages/vanilla/src/db/entities/index.ts | 3 + packages/vanilla/src/db/interfaces/DB.ts | 19 + packages/vanilla/src/db/interfaces/index.ts | 1 + .../vanilla/src/db/migrations/001-init.sql | 21 + .../src/db/repositories/DB.repository.ts | 40 + .../src/db/repositories/SQLDB.repository.ts | 41 + .../src/db/repositories/SQLite.repository.ts | 10 + .../repositories/adapters/SQLiteDB.adapter.ts | 30 + .../src/db/repositories/adapters/index.ts | 1 + .../adapters/interfaces/DB.adapter.ts | 36 + .../repositories/adapters/interfaces/index.ts | 1 + packages/vanilla/src/db/repositories/index.ts | 3 + packages/vanilla/src/db/utils/SQLBuilder.ts | 95 +++ packages/vanilla/src/events/EventEmitter.ts | 47 ++ packages/vanilla/src/events/index.ts | 1 + packages/vanilla/src/index.ts | 5 +- packages/vanilla/src/types.ts | 32 +- packages/vanilla/src/utils.types.ts | 61 -- .../vanilla/src/{utils.ts => utils/index.ts} | 2 +- packages/vanilla/src/utils/types.ts | 9 + .../account/src/XrplAccountIndexer.ts | 15 +- packages/xrpl/packages/account/src/types.ts | 4 +- .../ledgers/src/XrplLedgersIndexer.ts | 11 +- packages/xrpl/packages/ledgers/src/types.ts | 4 +- packages/xrpl/src/XrplIndexer.ts | 3 +- packages/xrpl/src/types.ts | 5 +- scripts/create-flavour-impl.js | 10 +- scripts/create-flavour.js | 9 +- scripts/drill.js | 5 +- scripts/utils/names.js | 4 - 49 files changed, 1202 insertions(+), 409 deletions(-) delete mode 100644 packages/vanilla/src/EventEmitter.ts delete mode 100644 packages/vanilla/src/IndexerState.repository.ts create mode 100644 packages/vanilla/src/db/SQLiteDB.ts create mode 100644 packages/vanilla/src/db/entities/Entity.ts create mode 100644 packages/vanilla/src/db/entities/LastEvent.ts create mode 100644 packages/vanilla/src/db/entities/PendingEvent.ts create mode 100644 packages/vanilla/src/db/entities/index.ts create mode 100644 packages/vanilla/src/db/interfaces/DB.ts create mode 100644 packages/vanilla/src/db/interfaces/index.ts create mode 100644 packages/vanilla/src/db/migrations/001-init.sql create mode 100644 packages/vanilla/src/db/repositories/DB.repository.ts create mode 100644 packages/vanilla/src/db/repositories/SQLDB.repository.ts create mode 100644 packages/vanilla/src/db/repositories/SQLite.repository.ts create mode 100644 packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts create mode 100644 packages/vanilla/src/db/repositories/adapters/index.ts create mode 100644 packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts create mode 100644 packages/vanilla/src/db/repositories/adapters/interfaces/index.ts create mode 100644 packages/vanilla/src/db/repositories/index.ts create mode 100644 packages/vanilla/src/db/utils/SQLBuilder.ts create mode 100644 packages/vanilla/src/events/EventEmitter.ts create mode 100644 packages/vanilla/src/events/index.ts delete mode 100644 packages/vanilla/src/utils.types.ts rename packages/vanilla/src/{utils.ts => utils/index.ts} (97%) create mode 100644 packages/vanilla/src/utils/types.ts diff --git a/.gitignore b/.gitignore index c74486a..830bba6 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ node_modules/ .vscode .DS_Store dist/ -drills/*/state \ No newline at end of file +drills/*/persistence \ No newline at end of file diff --git a/drills/ethers-typechain-contract/drill.ts b/drills/ethers-typechain-contract/drill.ts index 1baaa69..27cbaae 100644 --- a/drills/ethers-typechain-contract/drill.ts +++ b/drills/ethers-typechain-contract/drill.ts @@ -4,7 +4,8 @@ import { BridgeDoorCommon__factory } from "@peersyst/xrp-evm-contracts"; async function drill() { const indexer = new EthersTypechainContractIndexer("0x0FCCFB556B4aA1B44F31220AcDC8007D46514f31", BridgeDoorCommon__factory, { wsUrl: "ws://168.119.63.112:8546", - persistState: false, + persist: false, + persistenceFilePath: "persistence/.ethers-typechain-contract-indexer.db", }); await indexer.run(); } diff --git a/drills/xrpl-account/drill.ts b/drills/xrpl-account/drill.ts index ebf5299..4be10d1 100644 --- a/drills/xrpl-account/drill.ts +++ b/drills/xrpl-account/drill.ts @@ -3,8 +3,8 @@ import { XrplAccountIndexer } from "@bloxer/xrpl-account"; async function drill() { const indexer = new XrplAccountIndexer("rEAjhZHotzo2jqPbjFpAEacgwc5XoUppgo", { wsUrl: "wss://s.devnet.rippletest.net:51233", - stateFilePath: "state/.xrpl-account-indexer-state.json", - persistState: false, + persistenceFilePath: "persistence/.xrpl-account-indexer.db", + persist: false, startingBlock: "latest", }); indexer.on("XChainCreateBridge", (transaction) => { diff --git a/drills/xrpl-ledgers/drill.ts b/drills/xrpl-ledgers/drill.ts index 1ca0f3e..825d9b4 100644 --- a/drills/xrpl-ledgers/drill.ts +++ b/drills/xrpl-ledgers/drill.ts @@ -3,9 +3,9 @@ import { XrplLedgersIndexer } from "@bloxer/xrpl-ledgers"; async function drill() { const indexer = new XrplLedgersIndexer({ wsUrl: "wss://s1.ripple.com/", - stateFilePath: "state/.xrpl-blocks-indexer-state.json", + persistenceFilePath: "persistence/.xrpl-blocks-indexer.db", startingBlock: 1000000, - persistState: false, + persist: false, }); indexer.on("Ledger", (ledger) => { // eslint-disable-next-line no-console diff --git a/package-lock.json b/package-lock.json index 00d4515..b82badc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "packages/*/packages/*" ], "devDependencies": { + "@types/fs-extra": "^11.0.4", "@types/node": "^20.6.2", "@typescript-eslint/eslint-plugin": "^6.7.2", "@typescript-eslint/parser": "^6.7.2", @@ -19,7 +20,7 @@ "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.28.1", "eslint-plugin-prettier": "^5.0.0", - "fs-extra": "^11.1.1", + "fs-extra": "^11.2.0", "lerna": "^7.3.0", "prettier": "^3.0.3", "ts-node": "^10.9.1", @@ -990,6 +991,12 @@ "@ethersproject/strings": "^5.7.0" } }, + "node_modules/@gar/promisify": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz", + "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", + "optional": true + }, "node_modules/@humanwhocodes/config-array": { "version": "0.11.11", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.11.tgz", @@ -1378,6 +1385,35 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/@npmcli/move-file": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.1.2.tgz", + "integrity": "sha512-1SUf/Cg2GzGDyaf15aR9St9TWlb+XvbZXWpDx8YKs7MLzMH/BCeopv+y9vzrzgkfykCGuWOlSu3mZhj2+FQcrg==", + "deprecated": "This functionality has been moved to @npmcli/fs", + "optional": true, + "dependencies": { + "mkdirp": "^1.0.4", + "rimraf": "^3.0.2" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/@npmcli/move-file/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "optional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/@npmcli/node-gyp": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@npmcli/node-gyp/-/node-gyp-3.0.0.tgz", @@ -1939,6 +1975,15 @@ "integrity": "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==", "dev": true }, + "node_modules/@swisstype/essential": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@swisstype/essential/-/essential-0.0.5.tgz", + "integrity": "sha512-tM5Z+Dbib2MLBNe3BcxTkiQ14gkglWLnXR4loCn/JmGFBDkbe3FSIGMp6jcwbkXZBhKbFBMtI5UZW7rGTtt17w==", + "dev": true, + "peerDependencies": { + "typescript": "^5.x" + } + }, "node_modules/@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", @@ -2026,6 +2071,16 @@ "@types/node": "*" } }, + "node_modules/@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dev": true, + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "node_modules/@types/json-schema": { "version": "7.0.13", "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.13.tgz", @@ -2038,6 +2093,15 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, + "node_modules/@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/minimatch": { "version": "3.0.5", "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", @@ -2313,7 +2377,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", - "dev": true + "devOptional": true }, "node_modules/acorn": { "version": "8.10.0", @@ -2371,7 +2435,7 @@ "version": "4.5.0", "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", - "dev": true, + "devOptional": true, "dependencies": { "humanize-ms": "^1.2.1" }, @@ -2383,7 +2447,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", - "dev": true, + "devOptional": true, "dependencies": { "clean-stack": "^2.0.0", "indent-string": "^4.0.0" @@ -2436,7 +2500,7 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -2460,13 +2524,13 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==", - "dev": true + "devOptional": true }, "node_modules/are-we-there-yet": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-3.0.1.tgz", "integrity": "sha512-QZW4EDmGwlYur0Yyf/b2uGucHQMa8aFUP7eu9ddR73vvhFyt4V0Vl3QHPcTNJ8l6qYOBdxgXdnBXQrHilfRQBg==", - "dev": true, + "devOptional": true, "dependencies": { "delegates": "^1.0.0", "readable-stream": "^3.6.0" @@ -2678,7 +2742,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true + "devOptional": true }, "node_modules/base-x": { "version": "3.0.9", @@ -2776,7 +2840,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dev": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -2804,7 +2867,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, + "devOptional": true, "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -2849,7 +2912,6 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "dev": true, "funding": [ { "type": "github", @@ -3080,7 +3142,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-2.0.0.tgz", "integrity": "sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==", - "dev": true, "engines": { "node": ">=10" } @@ -3113,7 +3174,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", - "dev": true, + "devOptional": true, "engines": { "node": ">=6" } @@ -3245,7 +3306,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/color-support/-/color-support-1.1.3.tgz", "integrity": "sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg==", - "dev": true, + "devOptional": true, "bin": { "color-support": "bin.js" } @@ -3289,7 +3350,7 @@ "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true + "devOptional": true }, "node_modules/concat-stream": { "version": "2.0.0", @@ -3310,7 +3371,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "dev": true + "devOptional": true }, "node_modules/conventional-changelog-angular": { "version": "6.0.0", @@ -3593,12 +3654,34 @@ "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/dedent": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz", "integrity": "sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA==", "dev": true }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -3824,7 +3907,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "dev": true + "devOptional": true }, "node_modules/deprecation": { "version": "2.3.1", @@ -3841,6 +3924,14 @@ "node": ">=4" } }, + "node_modules/detect-libc": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.2.tgz", + "integrity": "sha512-UX6sGumvvqSaXgdKGUsgZWqcUyIXZ/vZTrlRT/iobiKhGL0zL4d3osHj3uqllWJK+i+sixDS/3COVEOFbupFyw==", + "engines": { + "node": ">=8" + } + }, "node_modules/diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -3966,13 +4057,12 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "devOptional": true }, "node_modules/encoding": { "version": "0.1.13", "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", - "dev": true, "optional": true, "dependencies": { "iconv-lite": "^0.6.2" @@ -3982,7 +4072,6 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", - "dev": true, "optional": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" @@ -3995,7 +4084,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "dependencies": { "once": "^1.4.0" } @@ -4016,7 +4104,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", "integrity": "sha512-+h1lkLKhZMTYjog1VEpJNG7NZJWcuc2DDk/qsqSTRRCOXiLjeQ1d1/udrUGhqMxUgAlwKNZ0cf2uqan5GLuS2A==", - "dev": true, + "devOptional": true, "engines": { "node": ">=6" } @@ -4037,7 +4125,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/err-code/-/err-code-2.0.3.tgz", "integrity": "sha512-2bmlRpNKBxT/CRmPOlyISQpNj+qSeYvcym/uT0Jx2bMOlKLtSy1ZmLuVxSEKKyor/N5yhvp/ZiG1oE3DEYMSFA==", - "dev": true + "devOptional": true }, "node_modules/error-ex": { "version": "1.3.2", @@ -4679,6 +4767,14 @@ "url": "https://github.com/sindresorhus/execa?sponsor=1" } }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "engines": { + "node": ">=6" + } + }, "node_modules/exponential-backoff": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", @@ -4964,14 +5060,12 @@ "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "dev": true + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.1.1", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.1.1.tgz", - "integrity": "sha512-MGIE4HOvQCeUCzmlHs0vXpih4ysz4wg9qiSAu6cd42lVwPbTM1TjV7RusoyQqMmk/95gdQZX72u+YW+c3eEpFQ==", - "dev": true, + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -5006,7 +5100,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "devOptional": true }, "node_modules/function-bind": { "version": "1.1.1", @@ -5044,7 +5138,7 @@ "version": "4.0.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-4.0.4.tgz", "integrity": "sha512-f9m+BEN5jkg6a0fZjleidjN51VE1X+mPFQ2DJ0uv1V39oCLCbsGe6yjbBnp7eK7z/+GAon99a3nHuqbuuthyPg==", - "dev": true, + "devOptional": true, "dependencies": { "aproba": "^1.0.3 || ^2.0.0", "color-support": "^1.1.3", @@ -5223,11 +5317,16 @@ "ini": "^1.3.2" } }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" + }, "node_modules/glob": { "version": "7.2.3", "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "dev": true, + "devOptional": true, "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -5259,7 +5358,7 @@ "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, + "devOptional": true, "dependencies": { "brace-expansion": "^1.1.7" }, @@ -5343,8 +5442,7 @@ "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/graphemer": { "version": "1.4.0", @@ -5462,7 +5560,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "dev": true + "devOptional": true }, "node_modules/hash-base": { "version": "3.1.0", @@ -5512,7 +5610,7 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==", - "dev": true + "devOptional": true }, "node_modules/http-proxy-agent": { "version": "5.0.0", @@ -5553,7 +5651,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "dev": true, + "devOptional": true, "dependencies": { "ms": "^2.0.0" } @@ -5679,7 +5777,7 @@ "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.8.19" } @@ -5688,16 +5786,22 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } }, + "node_modules/infer-owner": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/infer-owner/-/infer-owner-1.0.4.tgz", + "integrity": "sha512-IClj+Xz94+d7irH5qRyfJonOdfTzuDaifE6ZPWfx0N0+/ATZCbuTPq2prFl526urkQd90WyUKIh1DfBQ2hMz9A==", + "optional": true + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "dev": true, + "devOptional": true, "dependencies": { "once": "^1.3.0", "wrappy": "1" @@ -5711,8 +5815,7 @@ "node_modules/ini": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "dev": true + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==" }, "node_modules/init-package-json": { "version": "5.0.0", @@ -5828,7 +5931,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/ip/-/ip-2.0.0.tgz", "integrity": "sha512-WKa+XuLG1A1R0UWhl2+1XQSi+fZWMsYKffMZTTYsiZaUD8k2yDAj5atimTUD2TZkyCkNEeYE5NhFZmupOGtjYQ==", - "dev": true + "devOptional": true }, "node_modules/is-arguments": { "version": "1.1.1", @@ -5971,7 +6074,7 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=8" } @@ -6048,7 +6151,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", "integrity": "sha512-z7CMFGNrENq5iFB9Bqo64Xk6Y9sg+epq1myIcdHaGnbMTYOxvzsEtdYqQUylB7LxfkvgrrjP32T6Ywciio9UIQ==", - "dev": true + "devOptional": true }, "node_modules/is-nan": { "version": "1.3.2", @@ -6285,7 +6388,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true + "devOptional": true }, "node_modules/isobject": { "version": "3.0.1", @@ -6449,7 +6552,6 @@ "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", - "dev": true, "dependencies": { "universalify": "^2.0.0" }, @@ -6824,7 +6926,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7103,6 +7204,17 @@ "node": ">=6" } }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/min-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/min-indent/-/min-indent-1.0.1.tgz", @@ -7138,7 +7250,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, "funding": { "url": "https://github.com/sponsors/ljharb" } @@ -7170,7 +7281,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/minipass-collect/-/minipass-collect-1.0.2.tgz", "integrity": "sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==", - "dev": true, + "devOptional": true, "dependencies": { "minipass": "^3.0.0" }, @@ -7182,7 +7293,7 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7220,7 +7331,7 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/minipass-flush/-/minipass-flush-1.0.5.tgz", "integrity": "sha512-JmQSYYpPUqX5Jyn1mXaRwOda1uQ8HP5KAT/oDSLCzt1BYRhQU0/hDtsB1ufZfEEzMZ9aAVmsBw8+FWsIXlClWw==", - "dev": true, + "devOptional": true, "dependencies": { "minipass": "^3.0.0" }, @@ -7232,7 +7343,7 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7266,7 +7377,7 @@ "version": "1.2.4", "resolved": "https://registry.npmjs.org/minipass-pipeline/-/minipass-pipeline-1.2.4.tgz", "integrity": "sha512-xuIq7cIOt09RPRJ19gdi4b+RiNvDFYe5JH+ggNvBqGqpQXcru3PcRmOZuHBKWK1Txf9+cQ+HMVN4d6z46LZP7A==", - "dev": true, + "devOptional": true, "dependencies": { "minipass": "^3.0.0" }, @@ -7278,7 +7389,7 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7290,7 +7401,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/minipass-sized/-/minipass-sized-1.0.3.tgz", "integrity": "sha512-MbkQQ2CTiBMlA2Dm/5cY+9SWFEN8pzzOXi6rlM5Xxq0Yqbda5ZQy9sU75a673FE9ZK0Zsbr6Y5iP6u9nktfg2g==", - "dev": true, + "devOptional": true, "dependencies": { "minipass": "^3.0.0" }, @@ -7302,7 +7413,7 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, + "devOptional": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7314,7 +7425,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-2.1.2.tgz", "integrity": "sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==", - "dev": true, "dependencies": { "minipass": "^3.0.0", "yallist": "^4.0.0" @@ -7327,7 +7437,6 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -7339,7 +7448,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, "bin": { "mkdirp": "bin/cmd.js" }, @@ -7347,6 +7455,11 @@ "node": ">=10" } }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + }, "node_modules/modify-values": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/modify-values/-/modify-values-1.0.1.tgz", @@ -7400,6 +7513,11 @@ "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==" }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", + "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==" + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7410,7 +7528,7 @@ "version": "0.6.3", "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 0.6" } @@ -7421,6 +7539,17 @@ "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", "dev": true }, + "node_modules/node-abi": { + "version": "3.56.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.56.0.tgz", + "integrity": "sha512-fZjdhDOeRcaS+rcpve7XuwHBmktS1nS1gzgghwKUQQ8nTy2FdSDr6ZT8k6YhvlJeHmmQMYiT/IH9hfco5zeW2Q==", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/node-addon-api": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-3.2.1.tgz", @@ -7805,7 +7934,7 @@ "version": "6.0.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==", - "dev": true, + "devOptional": true, "dependencies": { "are-we-there-yet": "^3.0.0", "console-control-strings": "^1.1.0", @@ -8077,7 +8206,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, "dependencies": { "wrappy": "1" } @@ -8203,7 +8331,7 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", - "dev": true, + "devOptional": true, "dependencies": { "aggregate-error": "^3.0.0" }, @@ -8500,7 +8628,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -8596,6 +8724,31 @@ "node": ">=8" } }, + "node_modules/prebuild-install": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.2.tgz", + "integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", @@ -8677,13 +8830,13 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz", "integrity": "sha512-6zWPyEOFaQBJYcGMHBKTKJ3u6TBsnMFOIZSa6ce1e/ZrrsOlnHRHbabMjLiBYKp+n44X9eUI6VUPaukCXHuG4g==", - "dev": true + "devOptional": true }, "node_modules/promise-retry": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/promise-retry/-/promise-retry-2.0.1.tgz", "integrity": "sha512-y+WKFlBR8BGXnsNlIHFGPZmyDf3DFMoLhaflAnyZgV6rG6xu+JwesTo2Q9R6XwYmtmwAFCkAk3e35jEdoeh/3g==", - "dev": true, + "devOptional": true, "dependencies": { "err-code": "^2.0.2", "retry": "^0.12.0" @@ -8716,6 +8869,15 @@ "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", "dev": true }, + "node_modules/pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, "node_modules/punycode": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.0.tgz", @@ -8754,6 +8916,28 @@ "node": ">=8" } }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/rc/node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/react-is": { "version": "18.2.0", "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.2.0.tgz", @@ -9222,7 +9406,7 @@ "version": "0.12.0", "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 4" } @@ -9498,7 +9682,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "dev": true + "devOptional": true }, "node_modules/scrypt-js": { "version": "3.0.1", @@ -9509,7 +9693,6 @@ "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dev": true, "dependencies": { "lru-cache": "^6.0.0" }, @@ -9524,7 +9707,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz", "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", - "dev": true + "devOptional": true }, "node_modules/set-function-name": { "version": "2.0.1", @@ -9603,7 +9786,7 @@ "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true + "devOptional": true }, "node_modules/sigstore": { "version": "1.9.0", @@ -9624,6 +9807,49 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -9637,7 +9863,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", - "dev": true, + "devOptional": true, "engines": { "node": ">= 6.0.0", "npm": ">= 3.0.0" @@ -9647,7 +9873,7 @@ "version": "2.7.1", "resolved": "https://registry.npmjs.org/socks/-/socks-2.7.1.tgz", "integrity": "sha512-7maUZy1N7uo6+WVEX6psASxtNlKaNVMlGQKkG/63nEDdLOWNbiUMoLK7X4uYoLhQstau72mLgfEWcXcwsaHbYQ==", - "dev": true, + "devOptional": true, "dependencies": { "ip": "^2.0.0", "smart-buffer": "^4.2.0" @@ -9751,6 +9977,285 @@ "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", "dev": true }, + "node_modules/sqlite": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/sqlite/-/sqlite-5.1.1.tgz", + "integrity": "sha512-oBkezXa2hnkfuJwUo44Hl9hS3er+YFtueifoajrgidvqsJRQFpc5fKoAkAor1O5ZnLoa28GBScfHXs8j0K358Q==" + }, + "node_modules/sqlite3": { + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-5.1.7.tgz", + "integrity": "sha512-GGIyOiFaG+TUra3JIfkI/zGP8yZYLPQ0pl1bH+ODjiX57sPhrLU5sQJn1y9bDKZUFYkX1crlrPfSYt0BKKdkog==", + "hasInstallScript": true, + "dependencies": { + "bindings": "^1.5.0", + "node-addon-api": "^7.0.0", + "prebuild-install": "^7.1.1", + "tar": "^6.1.11" + }, + "optionalDependencies": { + "node-gyp": "8.x" + }, + "peerDependencies": { + "node-gyp": "8.x" + }, + "peerDependenciesMeta": { + "node-gyp": { + "optional": true + } + } + }, + "node_modules/sqlite3/node_modules/@npmcli/fs": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@npmcli/fs/-/fs-1.1.1.tgz", + "integrity": "sha512-8KG5RD0GVP4ydEzRn/I4BNDuxDtqVbOdm8675T49OIG/NGhaK0pjPX7ZcDlvKYbA+ulvVK3ztfcF4uBdOxuJbQ==", + "optional": true, + "dependencies": { + "@gar/promisify": "^1.0.1", + "semver": "^7.3.5" + } + }, + "node_modules/sqlite3/node_modules/@tootallnate/once": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-1.1.2.tgz", + "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", + "optional": true, + "engines": { + "node": ">= 6" + } + }, + "node_modules/sqlite3/node_modules/cacache": { + "version": "15.3.0", + "resolved": "https://registry.npmjs.org/cacache/-/cacache-15.3.0.tgz", + "integrity": "sha512-VVdYzXEn+cnbXpFgWs5hTT7OScegHVmLhJIR8Ufqk3iFD6A6j5iSX1KuBTfNEv4tdJWE2PzA6IVFtcLC7fN9wQ==", + "optional": true, + "dependencies": { + "@npmcli/fs": "^1.0.0", + "@npmcli/move-file": "^1.0.1", + "chownr": "^2.0.0", + "fs-minipass": "^2.0.0", + "glob": "^7.1.4", + "infer-owner": "^1.0.4", + "lru-cache": "^6.0.0", + "minipass": "^3.1.1", + "minipass-collect": "^1.0.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.2", + "mkdirp": "^1.0.3", + "p-map": "^4.0.0", + "promise-inflight": "^1.0.1", + "rimraf": "^3.0.2", + "ssri": "^8.0.1", + "tar": "^6.0.2", + "unique-filename": "^1.1.1" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/sqlite3/node_modules/fs-minipass": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", + "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", + "optional": true, + "dependencies": { + "minipass": "^3.0.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/sqlite3/node_modules/http-proxy-agent": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz", + "integrity": "sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==", + "optional": true, + "dependencies": { + "@tootallnate/once": "1", + "agent-base": "6", + "debug": "4" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/sqlite3/node_modules/make-fetch-happen": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/make-fetch-happen/-/make-fetch-happen-9.1.0.tgz", + "integrity": "sha512-+zopwDy7DNknmwPQplem5lAZX/eCOzSvSNNcSKm5eVwTkOBzoktEfXsa9L23J/GIRhxRsaxzkPEhrJEpE2F4Gg==", + "optional": true, + "dependencies": { + "agentkeepalive": "^4.1.3", + "cacache": "^15.2.0", + "http-cache-semantics": "^4.1.0", + "http-proxy-agent": "^4.0.1", + "https-proxy-agent": "^5.0.0", + "is-lambda": "^1.0.1", + "lru-cache": "^6.0.0", + "minipass": "^3.1.3", + "minipass-collect": "^1.0.2", + "minipass-fetch": "^1.3.2", + "minipass-flush": "^1.0.5", + "minipass-pipeline": "^1.2.4", + "negotiator": "^0.6.2", + "promise-retry": "^2.0.1", + "socks-proxy-agent": "^6.0.0", + "ssri": "^8.0.0" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/sqlite3/node_modules/minipass": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", + "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", + "optional": true, + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/sqlite3/node_modules/minipass-fetch": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/minipass-fetch/-/minipass-fetch-1.4.1.tgz", + "integrity": "sha512-CGH1eblLq26Y15+Azk7ey4xh0J/XfJfrCox5LDJiKqI2Q2iwOLOKrlmIaODiSQS8d18jalF6y2K2ePUm0CmShw==", + "optional": true, + "dependencies": { + "minipass": "^3.1.0", + "minipass-sized": "^1.0.3", + "minizlib": "^2.0.0" + }, + "engines": { + "node": ">=8" + }, + "optionalDependencies": { + "encoding": "^0.1.12" + } + }, + "node_modules/sqlite3/node_modules/node-addon-api": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz", + "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==", + "engines": { + "node": "^16 || ^18 || >= 20" + } + }, + "node_modules/sqlite3/node_modules/node-gyp": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", + "integrity": "sha512-olTJRgUtAb/hOXG0E93wZDs5YiJlgbXxTwQAFHyNlRsXQnYzUaF2aGgujZbw+hR8aF4ZG/rST57bWMWD16jr9w==", + "optional": true, + "dependencies": { + "env-paths": "^2.2.0", + "glob": "^7.1.4", + "graceful-fs": "^4.2.6", + "make-fetch-happen": "^9.1.0", + "nopt": "^5.0.0", + "npmlog": "^6.0.0", + "rimraf": "^3.0.2", + "semver": "^7.3.5", + "tar": "^6.1.2", + "which": "^2.0.2" + }, + "bin": { + "node-gyp": "bin/node-gyp.js" + }, + "engines": { + "node": ">= 10.12.0" + } + }, + "node_modules/sqlite3/node_modules/nopt": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nopt/-/nopt-5.0.0.tgz", + "integrity": "sha512-Tbj67rffqceeLpcRXrT7vKAN8CwfPeIBgM7E6iBkmKLV7bEMwpGgYLGv0jACUsECaa/vuxP0IjEont6umdMgtQ==", + "optional": true, + "dependencies": { + "abbrev": "1" + }, + "bin": { + "nopt": "bin/nopt.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/sqlite3/node_modules/rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "optional": true, + "dependencies": { + "glob": "^7.1.3" + }, + "bin": { + "rimraf": "bin.js" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/sqlite3/node_modules/socks-proxy-agent": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz", + "integrity": "sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==", + "optional": true, + "dependencies": { + "agent-base": "^6.0.2", + "debug": "^4.3.3", + "socks": "^2.6.2" + }, + "engines": { + "node": ">= 10" + } + }, + "node_modules/sqlite3/node_modules/ssri": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/ssri/-/ssri-8.0.1.tgz", + "integrity": "sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==", + "optional": true, + "dependencies": { + "minipass": "^3.1.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/sqlite3/node_modules/unique-filename": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz", + "integrity": "sha512-Vmp0jIp2ln35UTXuryvjzkjGdRyf9b2lTXuSYUiPmzRcl3FDtYqAwOnTJkAngD9SWhnoJzDbTKwaOrZ+STtxNQ==", + "optional": true, + "dependencies": { + "unique-slug": "^2.0.0" + } + }, + "node_modules/sqlite3/node_modules/unique-slug": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/unique-slug/-/unique-slug-2.0.2.tgz", + "integrity": "sha512-zoWr9ObaxALD3DOPfjPSqxt4fnZiWblxHIgeWqW8x7UqDzEtHEQLzji2cuJYQFCU6KmoJikOYAZlrTHHebjx2w==", + "optional": true, + "dependencies": { + "imurmurhash": "^0.1.4" + } + }, + "node_modules/sqlite3/node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "optional": true, + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, "node_modules/ssri": { "version": "9.0.1", "resolved": "https://registry.npmjs.org/ssri/-/ssri-9.0.1.tgz", @@ -9787,7 +10292,7 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, + "devOptional": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -9861,7 +10366,7 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, + "devOptional": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -9985,7 +10490,6 @@ "version": "6.1.11", "resolved": "https://registry.npmjs.org/tar/-/tar-6.1.11.tgz", "integrity": "sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==", - "dev": true, "dependencies": { "chownr": "^2.0.0", "fs-minipass": "^2.0.0", @@ -9998,11 +10502,26 @@ "node": ">= 10" } }, + "node_modules/tar-fs": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.1.tgz", + "integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-fs/node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", - "dev": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -10018,7 +10537,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-2.1.0.tgz", "integrity": "sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==", - "dev": true, "dependencies": { "minipass": "^3.0.0" }, @@ -10030,7 +10548,6 @@ "version": "3.3.6", "resolved": "https://registry.npmjs.org/minipass/-/minipass-3.3.6.tgz", "integrity": "sha512-DxiNidxSEK+tHG6zOIklvNOwm3hvCrbUrdtzY74U6HKTJxvIDfOUL5W5P2Ghd3DTkhhKPYGqeNUIh5qcM4YBfw==", - "dev": true, "dependencies": { "yallist": "^4.0.0" }, @@ -10304,6 +10821,17 @@ "node": "^14.17.0 || ^16.13.0 || >=18.0.0" } }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -10479,7 +11007,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", - "dev": true, "engines": { "node": ">= 10.0.0" } @@ -10654,7 +11181,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "dev": true, + "devOptional": true, "dependencies": { "string-width": "^1.0.2 || 2 || 3 || 4" } @@ -10708,8 +11235,7 @@ "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, "node_modules/write-file-atomic": { "version": "5.0.1", @@ -10889,8 +11415,7 @@ "node_modules/yallist": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", - "dev": true + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" }, "node_modules/yargs": { "version": "16.2.0", @@ -10965,7 +11490,14 @@ "version": "0.0.2", "license": "ISC", "dependencies": { + "fs-extra": "^11.2.0", + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7", "tslog": "^4.9.2" + }, + "devDependencies": { + "@swisstype/essential": "^0.0.5", + "@types/fs-extra": "^11.0.4" } }, "packages/xrpl": { diff --git a/package.json b/package.json index 9577b38..73f8558 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,8 @@ "clean": "lerna run --parallel clean", "drill": "node ./scripts/drill.js" }, - "dependencies": {}, "devDependencies": { + "@types/fs-extra": "^11.0.4", "@types/node": "^20.6.2", "@typescript-eslint/eslint-plugin": "^6.7.2", "@typescript-eslint/parser": "^6.7.2", @@ -25,7 +25,7 @@ "eslint-config-prettier": "^9.0.0", "eslint-plugin-import": "^2.28.1", "eslint-plugin-prettier": "^5.0.0", - "fs-extra": "^11.1.1", + "fs-extra": "^11.2.0", "lerna": "^7.3.0", "prettier": "^3.0.3", "ts-node": "^10.9.1", diff --git a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts index 2898a43..c35f208 100644 --- a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts +++ b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts @@ -1,10 +1,6 @@ import { EthersIndexer } from "@bloxer/ethers"; import { EthersTypechainContractIndexerEvents } from "./events"; -import { - EthersTypechainContractIndexerConfig, - EthersTypechainContractIndexerState, - EthersTypechainContractIndexerIndexOptions, -} from "./types"; +import { EthersTypechainContractIndexerConfig, EthersTypechainContractIndexerIndexOptions } from "./types"; import { EthersTypechainContractProvider } from "./EthersTypechainContractProvider"; import { TypechainContractFactory as GenericTypechainContractFactory, @@ -18,7 +14,6 @@ export class EthersTypechainContractIndexer; config: EthersTypechainContractIndexerConfig; - state: EthersTypechainContractIndexerState; indexOptions: EthersTypechainContractIndexerIndexOptions; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -28,7 +23,7 @@ export class EthersTypechainContractIndexer; -export type EthersTypechainContractIndexerState = ExtendedIndexerState<{}>; - export type EthersTypechainContractIndexerIndexOptions = ExtendedIndexOptions<{}>; diff --git a/packages/ethers/src/EthersIndexer.ts b/packages/ethers/src/EthersIndexer.ts index 18164c1..c42cefc 100644 --- a/packages/ethers/src/EthersIndexer.ts +++ b/packages/ethers/src/EthersIndexer.ts @@ -6,7 +6,6 @@ export abstract class EthersIndexer exte provider: Generics["provider"] extends undefined ? EthersProvider : Generics["provider"]; events: Generics["events"]; config: Generics["config"]; - state: Generics["state"]; indexOptions: Generics["indexOptions"]; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -16,7 +15,7 @@ export abstract class EthersIndexer exte logger: { name: "EthersIndexer", }, - stateFilePath: "./.ethers-indexer-state.json", + persistenceFilePath: "./.ethers-indexer.db", } as typeof this.defaultConfig); } diff --git a/packages/ethers/src/types.ts b/packages/ethers/src/types.ts index 752861b..6ac6dc4 100644 --- a/packages/ethers/src/types.ts +++ b/packages/ethers/src/types.ts @@ -1,16 +1,13 @@ -import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; +import { ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; import { EthersProvider } from "./EthersProvider"; export type EthersIndexerConfig = ExtendedIndexerConfig<{}>; -export type EthersIndexerState = ExtendedIndexerState<{}>; - export type EthersIndexerIndexOptions = ExtendedIndexOptions<{}>; export type EthersIndexerGenerics = { provider?: EthersProvider; events: Record any>; config?: EthersIndexerConfig; - state?: EthersIndexerState; indexOptions?: EthersIndexerIndexOptions; }; diff --git a/packages/vanilla/package.json b/packages/vanilla/package.json index 9479c2e..24585dc 100644 --- a/packages/vanilla/package.json +++ b/packages/vanilla/package.json @@ -11,11 +11,18 @@ "author": "Peersyst", "license": "ISC", "dependencies": { + "fs-extra": "^11.2.0", + "sqlite": "^5.1.1", + "sqlite3": "^5.1.7", "tslog": "^4.9.2" }, "sideEffects": false, "publishConfig": { "access": "public" }, - "gitHead": "fc337af75640624b77325644b54f4cad44d32f7d" + "gitHead": "fc337af75640624b77325644b54f4cad44d32f7d", + "devDependencies": { + "@swisstype/essential": "^0.0.5", + "@types/fs-extra": "^11.0.4" + } } diff --git a/packages/vanilla/src/EventEmitter.ts b/packages/vanilla/src/EventEmitter.ts deleted file mode 100644 index 45ae75d..0000000 --- a/packages/vanilla/src/EventEmitter.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { EventEmitter as Emitter } from "events"; - -export class EventEmitter any>> { - private emitter = new Emitter(); - - emit(event: Event, ...args: Parameters): boolean { - return this.emitter.emit(event as string, ...args); - } - - once(event: Event, listener: EventsDef[Event]): () => void { - this.emitter.once(event as string, listener); - return () => this.off(event, listener); - } - - on(event: Event, listener: EventsDef[Event]): () => void { - this.emitter.on(event as string, listener); - return () => this.off(event, listener); - } - - off(event: Event, listener: EventsDef[Event]): void { - this.emitter.off(event as string, listener); - } -} diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index ee8bc22..60e0c97 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -1,17 +1,18 @@ import { Logger } from "tslog"; import { withRetries, deepmerge } from "./utils"; -import { IndexerStateRepository } from "./IndexerState.repository"; -import { EventEmitter } from "./EventEmitter"; +import { EventEmitter } from "./events/EventEmitter"; import { - IndexOptions, IndexerGenerics, IndexerConfig, DefaultExtendedIndexerConfig, IndexerDefaultConfig, InheritedIndexerConfig, - InheritedIndexerState, + IndexOptions, } from "./types"; import { ProviderConstructor } from "./Provider"; +import { DB } from "./db/interfaces"; +import { SQLiteDB } from "./db/SQLiteDB"; +import { LastEvent } from "./db/entities"; export abstract class Indexer { private _defaultConfig: IndexerDefaultConfig = { @@ -25,8 +26,8 @@ export abstract class Indexer { prettyLogTemplate: "{{dd}}-{{mm}}-{{yyyy}} {{hh}}:{{MM}}:{{ss}}:{{ms}} {{name}} {{logLevelName}} ", prettyLogTimeZone: "local", }, - stateFilePath: ".bloxer-indexer.state.json", - persistState: true, + persistenceFilePath: ".bloxer-indexer.db", + persist: true, }; protected get defaultConfig(): DefaultExtendedIndexerConfig> { // Children will get the right type @@ -70,7 +71,10 @@ export abstract class Indexer { private _provider: Generics["provider"]; - private readonly indexerStateRepository: IndexerStateRepository>; + /** + * Reference to the database. + */ + private readonly db: DB; /** * The number of reconnect retries @@ -102,21 +106,6 @@ export abstract class Indexer { */ protected readonly emit: EventEmitter["emit"]; - /** - * Gets the state or a nested property from the state. - */ - protected readonly getState: IndexerStateRepository>["get"]; - - /** - * Sets the state. - */ - protected readonly setState: IndexerStateRepository>["set"]; - - /** - * Sets a partial state. - */ - protected readonly setPartialState: IndexerStateRepository>["setPartial"]; - /** * Requests the provider with retries * Acts as a wrapper for provider.request @@ -133,16 +122,10 @@ export abstract class Indexer { this.overrideDefaultConfig(this.defaultConfig as DefaultExtendedIndexerConfig>); this.config = deepmerge(this.defaultConfig, config) as typeof this.config; this.logger = new Logger(this.config.logger); - this.indexerStateRepository = new IndexerStateRepository({ - stateFilePath: this.config.stateFilePath, - persistState: this.config.persistState, - }); + this.db = new SQLiteDB(this.config.persistenceFilePath); this.on = this.eventEmitter.on.bind(this.eventEmitter); this.emit = this.eventEmitter.emit.bind(this.eventEmitter); - this.getState = this.indexerStateRepository.get.bind(this.indexerStateRepository); - this.setState = this.indexerStateRepository.set.bind(this.indexerStateRepository); - this.setPartialState = this.indexerStateRepository.setPartial.bind(this.indexerStateRepository); this.request = async function request(...args: any[]) { return this.withRetries( async () => { @@ -334,46 +317,50 @@ export abstract class Indexer { } /** - * Transforms the state into index options - * @param state The state - * @returns The index options + * Gets index options. + * @param nextBlock The next block. + * @param lastEvent The last event. */ - protected indexOptionsFromState(state: InheritedIndexerState): IndexOptions { - return { - startingBlock: state.block, - previousTransaction: state.transaction, - }; + private getIndexOptions(nextBlock: number | undefined, lastEvent: LastEvent | undefined): IndexOptions { + if (nextBlock) { + return { startingBlock: nextBlock }; + } else if (lastEvent) { + return { startingBlock: lastEvent.block, previousTransaction: lastEvent.hash }; + } else { + return { startingBlock: this.config.startingBlock === "latest" ? this.latestBlock : this.config.startingBlock }; + } } /** * Runs the indexer */ async run(): Promise { - await this.initializeProvider(); + await Promise.all([this.db.open(), this.initializeProvider()]); - let state = this.getState() as InheritedIndexerState; - let nextBlock; + // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s + let lastEvent = {} as any; + let nextBlock: number | undefined; this.running = true; while (this.running) { this.latestBlock = await this.latestBlockPromise; - if (nextBlock !== undefined ? nextBlock <= this.latestBlock : !state.block || state.block <= this.latestBlock) { - const fromBlock = - nextBlock ?? state.block ?? (this.config.startingBlock === "latest" ? this.latestBlock : this.config.startingBlock); - const indexingFrom = fromBlock ?? "genesys"; + + if (nextBlock !== undefined ? nextBlock <= this.latestBlock : !lastEvent.block || lastEvent.block <= this.latestBlock) { + const indexOptions = this.getIndexOptions(nextBlock, lastEvent); + const indexingFrom = indexOptions.startingBlock ?? "genesis"; + this.logger.info(`Indexing from block ${indexingFrom} to latest`); + try { - nextBlock = await this.index( - nextBlock !== undefined ? { startingBlock: nextBlock } : this.indexOptionsFromState({ block: fromBlock, ...state }), - ); - this.setState({ - block: nextBlock, - } as InheritedIndexerState); + nextBlock = await this.index(indexOptions); + this.logger.info(`Indexed blocks from ${indexingFrom} to ${nextBlock - 1} (latest)`); } catch (e) { this.logger.error(`Indexing from block ${indexingFrom} to latest failed with error ${e}`); this.logger.info(`Retrying indexing from block ${indexingFrom} to latest...`); - state = this.getState() as InheritedIndexerState; + + // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s + lastEvent = {}; nextBlock = undefined; } } @@ -386,8 +373,10 @@ export abstract class Indexer { async stop(): Promise { if (this.running) { this.logger.info("Stopping indexer..."); + this.running = false; - await this.unsubscribeFromLatestBlock(); + await Promise.all([this.db.close(), this.unsubscribeFromLatestBlock()]); + this.logger.info("Indexer stopped"); } else { this.logger.warn("Indexer is already stopped"); diff --git a/packages/vanilla/src/IndexerState.repository.ts b/packages/vanilla/src/IndexerState.repository.ts deleted file mode 100644 index aeddc79..0000000 --- a/packages/vanilla/src/IndexerState.repository.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { existsSync, readFileSync, writeFileSync } from "fs"; -import { getAttribute } from "./utils"; -import type { IndexerState } from "./types"; -import type { DeepPick, NestedKeys } from "./utils.types"; - -export type IndexerStateRepositoryOptions = { - stateFilePath: string; - persistState: boolean; -}; - -export class IndexerStateRepository { - /** - * Allows to cache the state in memory. - */ - private currentState: State; - - private readonly stateFilePath: string; - private readonly persistState: boolean; - - constructor({ stateFilePath, persistState }: IndexerStateRepositoryOptions) { - this.stateFilePath = stateFilePath; - this.persistState = persistState; - } - - /** - * Reads the state from the file or returns an initial state if the file does not exist. - * @returns The state. - */ - private readState(): State { - if (existsSync(this.stateFilePath)) return JSON.parse(readFileSync(this.stateFilePath).toString()) as State; - else return {} as State; - } - - /** - * Writes the state to the file. - * @param state The state to write. - */ - private writeState(state: State): void { - return writeFileSync(this.stateFilePath, JSON.stringify(state)); - } - - /** - * Gets the state or a nested property from the state. - * @param key An optional nested key - * @returns The state or a nested property from the state. - */ - get | void = void>(key?: Key): Key extends NestedKeys ? DeepPick : State { - if (!this.currentState) this.currentState = this.readState(); - - let result; - - if (key) result = getAttribute(this.currentState, key as NestedKeys); - else result = this.currentState; - - return result as Key extends NestedKeys ? DeepPick : State; - } - - /** - * Sets the state. - * @param state The state to set. - */ - set(state: State): void { - if (this.persistState) { - this.writeState(state); - this.currentState = state; - } - } - - /** - * Sets a partial state. - * @param state The partial state to set. - */ - setPartial(state: Partial): void { - if (this.persistState) { - this.set({ ...this.get(), ...state } as State); - } - } -} diff --git a/packages/vanilla/src/Provider.ts b/packages/vanilla/src/Provider.ts index b0b7c21..81785ef 100644 --- a/packages/vanilla/src/Provider.ts +++ b/packages/vanilla/src/Provider.ts @@ -1,4 +1,4 @@ -import { EventEmitter } from "./EventEmitter"; +import { EventEmitter } from "./events/EventEmitter"; export type ProviderEvents = { connected: () => void; diff --git a/packages/vanilla/src/db/SQLiteDB.ts b/packages/vanilla/src/db/SQLiteDB.ts new file mode 100644 index 0000000..b50dc38 --- /dev/null +++ b/packages/vanilla/src/db/SQLiteDB.ts @@ -0,0 +1,46 @@ +import { cached as sqliteCached } from "sqlite3"; +import { open as openSQLite, Database } from "sqlite"; +import { SQLiteRepository } from "./repositories"; +import { EntityConstructor } from "./entities"; +import { exists, outputFile } from "fs-extra"; +import { DB } from "./interfaces"; + +/** + * SQLite Database implementation of DB + */ +export class SQLiteDB implements DB { + private _db: Database | undefined; + private get db(): Database { + if (!this._db) throw new Error(`SQLite Database ${this.filename} not initialized`); + return this._db; + } + private set db(db: Database | undefined) { + this._db = db; + } + + constructor(private readonly filename) {} + + async open(): Promise { + // Check if the database file exists, if not create it + const existsDB = await exists(this.filename); + if (!existsDB) outputFile(this.filename, ""); + + this.db = await openSQLite({ + filename: this.filename, + driver: sqliteCached.Database, + }); + + await this.db.migrate({ + migrationsPath: `${__dirname}/migrations`, + }); + } + + async close(): Promise { + await this.db.close(); + this.db = undefined; + } + + getRepository(entity: Entity): SQLiteRepository { + return new SQLiteRepository(this.db, entity); + } +} diff --git a/packages/vanilla/src/db/entities/Entity.ts b/packages/vanilla/src/db/entities/Entity.ts new file mode 100644 index 0000000..42c3524 --- /dev/null +++ b/packages/vanilla/src/db/entities/Entity.ts @@ -0,0 +1,41 @@ +import { AnyObject } from "@swisstype/essential"; + +export interface EntityConstructor { + // The entity type is set to `any` since the entity class is not known at compile time. + // A constructor function must be defined in the interface so TS understands that another class can extend it. + new (): any; + table: string; + fromRow(row: AnyObject): any; + toRow(entity: any): AnyObject; +} + +/** + * Creates an entity class that has to be extended by all entities. Similar to a decorator. + * @param table The table name of the entity. + */ +export function Entity(table: string): EntityConstructor { + return class Entity { + // Workaround to make the class abstract. Cannot return an abstract class from a function -.-. + protected constructor() {} + + static readonly table = table; + + /** + * Parse row into entity. + * Can be overridden by the extending class to parse complex values. + * @param row The row to parse. + */ + static fromRow(row: AnyObject): Entity { + return row; + } + + /** + * Transform entity into row. + * Can be overridden by the extending class to parse complex values. + * @param entity The entity to transform. + */ + static toRow(entity: Entity): AnyObject { + return entity; + } + } as EntityConstructor; // Has to be casted since the constructor is protected +} diff --git a/packages/vanilla/src/db/entities/LastEvent.ts b/packages/vanilla/src/db/entities/LastEvent.ts new file mode 100644 index 0000000..116d947 --- /dev/null +++ b/packages/vanilla/src/db/entities/LastEvent.ts @@ -0,0 +1,6 @@ +import { Entity } from "./Entity"; + +export class LastEvent extends Entity("last_event") { + hash: string; + block: number; +} diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts new file mode 100644 index 0000000..2067327 --- /dev/null +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -0,0 +1,24 @@ +import { AnyObject } from "@swisstype/essential"; +import { Entity } from "./Entity"; + +export class PendingEvent extends Entity("pending_event") { + hash: string; + block: number; + data: AnyObject; + + static fromRow(row: AnyObject): PendingEvent { + return { + hash: row.hash, + block: row.block, + data: JSON.parse(row.data), + }; + } + + static toRow(row: PendingEvent): AnyObject { + return { + hash: row.hash, + block: row.block, + data: row.data ? JSON.stringify(row.data) : undefined, + }; + } +} diff --git a/packages/vanilla/src/db/entities/index.ts b/packages/vanilla/src/db/entities/index.ts new file mode 100644 index 0000000..8b80b73 --- /dev/null +++ b/packages/vanilla/src/db/entities/index.ts @@ -0,0 +1,3 @@ +export * from "./LastEvent"; +export * from "./PendingEvent"; +export * from "./Entity"; diff --git a/packages/vanilla/src/db/interfaces/DB.ts b/packages/vanilla/src/db/interfaces/DB.ts new file mode 100644 index 0000000..7d3b3d5 --- /dev/null +++ b/packages/vanilla/src/db/interfaces/DB.ts @@ -0,0 +1,19 @@ +import { DBRepository } from "../repositories"; +import { EntityConstructor } from "../entities"; + +export interface DB { + /** + * Opens the database connection. + */ + open(): Promise; + + /** + * Closes the database connection. + */ + close(): Promise; + + /** + * Returns a repository for the given entity. + */ + getRepository(entity: Entity): DBRepository; +} diff --git a/packages/vanilla/src/db/interfaces/index.ts b/packages/vanilla/src/db/interfaces/index.ts new file mode 100644 index 0000000..ea09edf --- /dev/null +++ b/packages/vanilla/src/db/interfaces/index.ts @@ -0,0 +1 @@ +export * from "./DB"; diff --git a/packages/vanilla/src/db/migrations/001-init.sql b/packages/vanilla/src/db/migrations/001-init.sql new file mode 100644 index 0000000..388a4d2 --- /dev/null +++ b/packages/vanilla/src/db/migrations/001-init.sql @@ -0,0 +1,21 @@ +-------------------------------------------------------------------------------- +-- Up +-------------------------------------------------------------------------------- + +CREATE TABLE pending_event ( + hash TEXT PRIMARY KEY, + block INTEGER NOT NULL, + data TEXT NOT NULL +); + +CREATE TABLE last_event ( + hash TEXT PRIMARY KEY, + block INTEGER NOT NULL +); + +-------------------------------------------------------------------------------- +-- Down +-------------------------------------------------------------------------------- + +DROP TABLE pending_event; +DROP TABLE last_event; diff --git a/packages/vanilla/src/db/repositories/DB.repository.ts b/packages/vanilla/src/db/repositories/DB.repository.ts new file mode 100644 index 0000000..1f57009 --- /dev/null +++ b/packages/vanilla/src/db/repositories/DB.repository.ts @@ -0,0 +1,40 @@ +import { EntityConstructor } from "../entities"; +import { InstanceOf } from "../../utils/types"; + +export abstract class DBRepository { + constructor(protected readonly entity: Entity) {} + + /** + * Finds one resource from the DB. + * @param where The where clauses (will be joined with OR). + * @returns The resource or undefined if no resource is found. + */ + abstract findOne(...where: Partial>[]): Promise>; + + /** + * Returns all resources from the DB matching the given wheres. + * @param where The where clauses (will be joined with OR). + * @returns The found resources. + */ + abstract findAll(...where: Partial>[]): Promise[]>; + + /** + * Creates a resource in the DB. + * @param data The data to insert. + * @returns The created resource. + */ + abstract create(data: InstanceOf): Promise>; + + /** + * Updates a resource in the DB. + * @param data The data to update. + * @param where The where clauses (will be joined with OR). + */ + abstract update(data: Partial>, ...where: Partial>[]): Promise; + + /** + * Deletes a resource from the DB. + * @param where The where clauses (will be joined with OR). + */ + abstract delete(...where: Partial>[]): Promise; +} diff --git a/packages/vanilla/src/db/repositories/SQLDB.repository.ts b/packages/vanilla/src/db/repositories/SQLDB.repository.ts new file mode 100644 index 0000000..1aafd17 --- /dev/null +++ b/packages/vanilla/src/db/repositories/SQLDB.repository.ts @@ -0,0 +1,41 @@ +import { EntityConstructor } from "../entities"; +import { InstanceOf } from "../../utils/types"; +import { DBRepository } from "./DB.repository"; +import { SQLBuilder } from "../utils/SQLBuilder"; +import { DBAdapter } from "./adapters/interfaces"; + +export abstract class SQLDBRepository extends DBRepository { + private readonly sqlBuilder: SQLBuilder; + + constructor( + protected readonly db: DBAdapter, + entity: Entity, + ) { + super(entity); + + this.sqlBuilder = new SQLBuilder(entity); + } + + async findOne(...where: Partial>[]): Promise> { + const row = await this.db.get(this.sqlBuilder.buildSelect(...where)); + return row ? this.entity.fromRow(row) : undefined; + } + + async findAll(...where: Partial>[]): Promise[]> { + const rows = await this.db.all(this.sqlBuilder.buildSelect(...where)); + return rows.map((row) => this.entity.fromRow(row)); + } + + async create(data: InstanceOf): Promise> { + await this.db.create(this.sqlBuilder.buildInsert(data)); + return data; + } + + async update(data: Partial>, ...where: Partial>[]): Promise { + await this.db.update(this.sqlBuilder.buildSetClause(data, ...where)); + } + + async delete(...where: Partial>[]): Promise { + await this.db.delete(this.sqlBuilder.buildDelete(...where)); + } +} diff --git a/packages/vanilla/src/db/repositories/SQLite.repository.ts b/packages/vanilla/src/db/repositories/SQLite.repository.ts new file mode 100644 index 0000000..7517a9a --- /dev/null +++ b/packages/vanilla/src/db/repositories/SQLite.repository.ts @@ -0,0 +1,10 @@ +import { Database } from "sqlite"; +import { EntityConstructor } from "../entities"; +import { SQLDBRepository } from "./SQLDB.repository"; +import { SQLiteDBAdapter } from "./adapters"; + +export class SQLiteRepository extends SQLDBRepository { + constructor(db: Database, entity: Entity) { + super(new SQLiteDBAdapter(db), entity); + } +} diff --git a/packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts b/packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts new file mode 100644 index 0000000..967da32 --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts @@ -0,0 +1,30 @@ +import { AnyObject } from "@swisstype/essential"; +import { DBAdapter } from "./interfaces"; +import { Database } from "sqlite"; + +/** + * Implements the `DBAdapter` for an SQLite database. + */ +export class SQLiteDBAdapter implements DBAdapter { + constructor(private readonly db: Database) {} + + get(query: string): Promise { + return this.db.get(query); + } + + all(query: string): Promise { + return this.db.all(query); + } + + async create(query: string): Promise { + await this.db.run(query); + } + + async update(query: string): Promise { + await this.db.run(query); + } + + async delete(query: string): Promise { + await this.db.run(query); + } +} diff --git a/packages/vanilla/src/db/repositories/adapters/index.ts b/packages/vanilla/src/db/repositories/adapters/index.ts new file mode 100644 index 0000000..8067714 --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/index.ts @@ -0,0 +1 @@ +export * from "./SQLiteDB.adapter"; diff --git a/packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts b/packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts new file mode 100644 index 0000000..23b169e --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts @@ -0,0 +1,36 @@ +import { AnyObject } from "@swisstype/essential"; + +/** + * DB adapters are used by repositories to interact with the database. + */ +export interface DBAdapter { + /** + * Returns a single resource from the database. + * @param query The get query to execute. + */ + get(query: string): Promise; + + /** + * Returns a list of resources from the database. + * @param query The get query to execute. + */ + all(query: string): Promise; + + /** + * Creates a resource in the database. + * @param query The create query to execute. + */ + create(query: string): Promise; + + /** + * Updates a resource in the database. + * @param query The update query to execute. + */ + update(query: string): Promise; + + /** + * Deletes a resource from the database. + * @param query The delete query to execute. + */ + delete(query: string): Promise; +} diff --git a/packages/vanilla/src/db/repositories/adapters/interfaces/index.ts b/packages/vanilla/src/db/repositories/adapters/interfaces/index.ts new file mode 100644 index 0000000..1dd761a --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/interfaces/index.ts @@ -0,0 +1 @@ +export * from "./DB.adapter"; diff --git a/packages/vanilla/src/db/repositories/index.ts b/packages/vanilla/src/db/repositories/index.ts new file mode 100644 index 0000000..cedf55c --- /dev/null +++ b/packages/vanilla/src/db/repositories/index.ts @@ -0,0 +1,3 @@ +export * from "./DB.repository"; +export * from "./SQLDB.repository"; +export * from "./SQLite.repository"; diff --git a/packages/vanilla/src/db/utils/SQLBuilder.ts b/packages/vanilla/src/db/utils/SQLBuilder.ts new file mode 100644 index 0000000..68dbc59 --- /dev/null +++ b/packages/vanilla/src/db/utils/SQLBuilder.ts @@ -0,0 +1,95 @@ +import { EntityConstructor } from "../entities"; +import { InstanceOf } from "../../utils/types"; + +/** + * SQL builder for an entity. + */ +export class SQLBuilder { + constructor(protected readonly entity: Entity) {} + + /** + * Builds a where clause. + * @param where The where clauses (will be joined with OR). + * @returns The where clause or undefined if no where clauses are provided. + */ + buildWhereClause(...where: Partial>[]): string | undefined { + if (where.length === 0) return undefined; + + return `WHERE ${where + .map((whereGroup) => + Object.entries( + this.entity.toRow( + // Casted to `InstanceOf`, `Entity.toRow` will have to handle possible undefined values + whereGroup as InstanceOf, + ), + ) + .map((key, value) => `${key} = ${value}`) + .join(" AND "), + ) + .join(" OR ")}`; + } + + /** + * Adds a where clause to a query. + * @param query The query. + * @param where The where clauses (will be joined with OR). + * @returns The resulting query. + */ + withWhereClause(query: string, ...where: Partial>[]): string { + const whereClause = this.buildWhereClause(...where); + return whereClause ? `${query} ${whereClause}` : query; + } + + /** + * Builds a select query. + * @param where The where clauses (will be joined with OR). + * @returns The select query. + */ + buildSelect(...where: Partial>[]): string { + return this.withWhereClause( + `SELECT * + FROM ${this.entity.table}`, + ...where, + ); + } + + /** + * Builds an insert query. + * @param data The data to insert. + * @returns The insert query. + */ + buildInsert(data: InstanceOf): string { + return `INSERT INTO ${this.entity.table} (${Object.keys(data).join(", ")}) + VALUES (${Object.values(this.entity.toRow(data)).join(", ")})`; + } + + /** + * Builds an update query. + * @param data The data to update. + * @param where The where clauses (will be joined with OR). + * @returns The update query. + */ + buildSetClause(data: Partial>, ...where: Partial>[]): string { + return this.withWhereClause( + `UPDATE ${this.entity.table} + SET ${Object.entries( + this.entity.toRow( + // Casted to `InstanceOf`, `Entity.toRow` will have to handle possible undefined values + data as InstanceOf, + ), + ) + .map((key, value) => `${key} = ${value}`) + .join(", ")}`, + ...where, + ); + } + + /** + * Builds a delete query. + * @param where The where clauses (will be joined with OR). + * @returns The delete query. + */ + buildDelete(...where: Partial>[]): string { + return this.withWhereClause(`DELETE FROM ${this.entity.table}`, ...where); + } +} diff --git a/packages/vanilla/src/events/EventEmitter.ts b/packages/vanilla/src/events/EventEmitter.ts new file mode 100644 index 0000000..73a9973 --- /dev/null +++ b/packages/vanilla/src/events/EventEmitter.ts @@ -0,0 +1,47 @@ +import { EventEmitter as Emitter } from "events"; + +export class EventEmitter any>> { + private emitter = new Emitter(); + + /** + * Emits an event with the given arguments. + * @param event The event to emit. + * @param args The arguments to pass to the event listeners. + * @returns Returns `true` if the event had listeners, `false` otherwise. + */ + emit(event: Event, ...args: Parameters): boolean { + return this.emitter.emit(event as string, ...args); + } + + /** + * Adds a **one-time**`listener` function for the specified event. The + * next time `event` is triggered, this listener is removed and then invoked. + * @param event The event. + * @param listener The callback function + * @returns A function that removes the listener. + */ + once(event: Event, listener: EventsDef[Event]): () => void { + this.emitter.once(event as string, listener); + return () => this.off(event, listener); + } + + /** + * Adds the `listener` function to the end of the listeners array for the specified event. + * @param event The event. + * @param listener The callback function. + * @returns A function that removes the listener. + */ + on(event: Event, listener: EventsDef[Event]): () => void { + this.emitter.on(event as string, listener); + return () => this.off(event, listener); + } + + /** + * Removes the specified `listener` from the listener array for the specified event. + * @param event The event. + * @param listener The callback function. + */ + off(event: Event, listener: EventsDef[Event]): void { + this.emitter.off(event as string, listener); + } +} diff --git a/packages/vanilla/src/events/index.ts b/packages/vanilla/src/events/index.ts new file mode 100644 index 0000000..18b60ac --- /dev/null +++ b/packages/vanilla/src/events/index.ts @@ -0,0 +1 @@ +export * from "./EventEmitter"; diff --git a/packages/vanilla/src/index.ts b/packages/vanilla/src/index.ts index e934b3b..3c951f3 100644 --- a/packages/vanilla/src/index.ts +++ b/packages/vanilla/src/index.ts @@ -1,7 +1,6 @@ export * from "./Indexer"; export * from "./types"; export * from "./Provider"; -export * from "./EventEmitter"; -export * from "./IndexerState.repository"; +export * from "./events"; export * from "./utils"; -export * from "./utils.types"; +export * from "./utils/types"; diff --git a/packages/vanilla/src/types.ts b/packages/vanilla/src/types.ts index c722b1f..5e423b8 100644 --- a/packages/vanilla/src/types.ts +++ b/packages/vanilla/src/types.ts @@ -1,4 +1,5 @@ -import { AnyObject, Inherited, OmitRequired } from "./utils.types"; +import { Inherited } from "./utils/types"; +import { AnyObject, OmitRequired } from "@swisstype/essential"; import { Provider } from "./Provider"; import { ISettingsParam as LoggerConfig } from "tslog"; @@ -36,35 +37,19 @@ export type IndexerConfig = { */ logger?: LoggerConfig; /** - * The path to the state file. - * @default "./bloxer.state.json" + * The path to the persistence file. + * @default "./bloxer.db" */ - stateFilePath?: string; + persistenceFilePath?: string; /** - * Whether to persist the state to the state file. + * Whether to persist enable persistence. * @default true */ - persistState?: boolean; + persist?: boolean; }; export type IndexerDefaultConfig = Required>; -export type IndexerState = { - /** - * The index of the last indexed block. - */ - block?: number; - /** - * The hash of the last indexed transaction. - */ - transaction?: string; -}; - -export type ExtendedIndexerState = (ExtendedState extends undefined - ? {} - : Partial) & - IndexerState; - export type IndexOptions = { /** * The starting block to index from. @@ -96,8 +81,6 @@ export type DefaultExtendedIndexerConfig = Required>; -export type InheritedIndexerState = Inherited; - export type InheritedIndexOptions = Inherited; export type InheritedIndexerConfig = Inherited; @@ -108,6 +91,5 @@ export type IndexerGenerics = { provider: Provider; events: Record any>; config?: ExtendedIndexerConfig; - state?: ExtendedIndexerState; indexOptions?: ExtendedIndexOptions | undefined; }; diff --git a/packages/vanilla/src/utils.types.ts b/packages/vanilla/src/utils.types.ts deleted file mode 100644 index a3e5316..0000000 --- a/packages/vanilla/src/utils.types.ts +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Difference between A and B - */ -export type Difference = Omit; - -/** - * Omit required properties from T - */ -export type OmitRequired = { - [K in keyof T as Pick extends Pick, K> ? never : K]: T[K]; -}; - -/** - * Omit optional properties from T - */ -export type OmitOptional = { - [K in keyof T as Pick extends Pick, K> ? K : never]: T[K]; -}; - -/** - * All possible iterations for a recursive type - */ -export type MaxRecursiveIterations = 10; -// prettier-ignore -export type Iterations = [never, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; - -export type AnyObject = Record; - -/** - * Get nested keys from T in the form of key1.key2... - */ -export type NestedKeys = CoreNestedKeys; -export type CoreNestedKeys = I extends 0 - ? never - : { - [Key in keyof T]: T[Key] extends AnyObject - ? `${Exclude}` | `${Exclude}.${CoreNestedKeys}` - : Key; - }[Extract]; - -/** - * Pick K types from T with keys in the form of key1.key2... - */ -export type DeepPick> = CoreDeepPick; -export type CoreDeepPick = I extends 0 - ? never - : K extends `${infer FirstKey}.${infer RestKey}` - ? CoreDeepPick - : T[K]; - -/** - * If E inherits from T, return T & E, else return T - */ -export type Inherited = E extends T ? T & E : T; - -/** - * Makes all properties, included nested ones, partial - */ -export type DeepPartial = { - [P in keyof T]?: T[P] extends (infer U)[] ? DeepPartial[] : T[P] extends object ? DeepPartial : T[P]; -}; diff --git a/packages/vanilla/src/utils.ts b/packages/vanilla/src/utils/index.ts similarity index 97% rename from packages/vanilla/src/utils.ts rename to packages/vanilla/src/utils/index.ts index baf07da..eb27369 100644 --- a/packages/vanilla/src/utils.ts +++ b/packages/vanilla/src/utils/index.ts @@ -1,4 +1,4 @@ -import { DeepPick, NestedKeys } from "./utils.types"; +import { DeepPick, NestedKeys } from "@swisstype/essential"; /** * Executes a function with retries diff --git a/packages/vanilla/src/utils/types.ts b/packages/vanilla/src/utils/types.ts new file mode 100644 index 0000000..0c7a74b --- /dev/null +++ b/packages/vanilla/src/utils/types.ts @@ -0,0 +1,9 @@ +/** + * If E inherits from T, return T & E, else return T + */ +export type Inherited = E extends T ? T & E : T; + +/** + * Instance of a constructor + */ +export type InstanceOf = T extends { new (...params: any[]): infer I } ? I : never; diff --git a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts index 5c60b7d..56dade4 100644 --- a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts +++ b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts @@ -1,6 +1,6 @@ import { XrplIndexer, XrplProvider } from "@bloxer/xrpl"; import { XrplAccountIndexerEvents } from "./events"; -import { XrplAccountIndexerConfig, XrplAccountIndexerState, XrplAccountIndexerIndexOptions } from "./types"; +import { XrplAccountIndexerConfig, XrplAccountIndexerIndexOptions } from "./types"; import { isValidAddress } from "xrpl"; import { AccountTransaction } from "./xrpl.types"; @@ -8,7 +8,6 @@ export class XrplAccountIndexer extends XrplIndexer<{ provider: XrplProvider; events: XrplAccountIndexerEvents; config: XrplAccountIndexerConfig; - state: XrplAccountIndexerState; indexOptions: XrplAccountIndexerIndexOptions; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -18,7 +17,7 @@ export class XrplAccountIndexer extends XrplIndexer<{ logger: { name: "XrplAccountIndexer", }, - stateFilePath: "./.xrpl-account-indexer-state.json", + persistenceFilePath: "./.xrpl-account-indexer.db", transactionsBatchSize: 10000, } as typeof this.defaultConfig); } @@ -68,11 +67,11 @@ export class XrplAccountIndexer extends XrplIndexer<{ this.emit("Transaction", correctlyCastedAccountTx); this.emit(tx.TransactionType, correctlyCastedAccountTx as AccountTransaction); // Save the last indexed transaction state - this.setPartialState({ - transaction: tx.hash, - block: tx.ledger_index, - }); - // TODO: What happens if some handlers can treat the tx but others can't (Edge case) + // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=9d8b6213e44d4c1e8f2c805565bb2486&pm=s + // this.setPartialState({ + // transaction: tx.hash, + // block: tx.ledger_index, + // }); } else if (tx.hash === previousTransaction) { reachedPreviousTransaction = true; } diff --git a/packages/xrpl/packages/account/src/types.ts b/packages/xrpl/packages/account/src/types.ts index f93f158..abbcc00 100644 --- a/packages/xrpl/packages/account/src/types.ts +++ b/packages/xrpl/packages/account/src/types.ts @@ -1,4 +1,4 @@ -import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; +import { ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; export type XrplAccountIndexerConfig = ExtendedIndexerConfig<{ /** @@ -8,6 +8,4 @@ export type XrplAccountIndexerConfig = ExtendedIndexerConfig<{ transactionsBatchSize?: number; }>; -export type XrplAccountIndexerState = ExtendedIndexerState<{}>; - export type XrplAccountIndexerIndexOptions = ExtendedIndexOptions<{}>; diff --git a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts index f2eeb29..d35e70e 100644 --- a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts +++ b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts @@ -1,12 +1,11 @@ import { XrplIndexer, XrplProvider } from "@bloxer/xrpl"; import { XrplLedgersIndexerEvents } from "./events"; -import { SelectedLedgerType, XrplLedgersIndexerConfig, XrplLedgersIndexerIndexOptions, XrplLedgersIndexerState } from "./types"; +import { SelectedLedgerType, XrplLedgersIndexerConfig, XrplLedgersIndexerIndexOptions } from "./types"; export class XrplLedgersIndexer extends XrplIndexer<{ provider: XrplProvider; events: XrplLedgersIndexerEvents; config: XrplLedgersIndexerConfig; - state: XrplLedgersIndexerState; indexOptions: XrplLedgersIndexerIndexOptions; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -16,7 +15,7 @@ export class XrplLedgersIndexer extends XrplIndexer<{ logger: { name: "XrplLedgersIndexer", }, - stateFilePath: "./.xrpl-ledgers-indexer-state.json", + persistenceFilePath: "./.xrpl-ledgers-indexer.db", } as typeof this.defaultConfig); } @@ -39,6 +38,12 @@ export class XrplLedgersIndexer extends XrplIndexer<{ this.emit("Ledger", res.result.ledger as SelectedLedgerType); ++currentBlock; } + + // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=9d8b6213e44d4c1e8f2c805565bb2486&pm=s + // this.setState({ + // hash: res.result.ledger.ledger_hash + // block: res.result.ledger.ledger_index, + // } as InheritedIndexerState); } const nextLedger = currentBlock; diff --git a/packages/xrpl/packages/ledgers/src/types.ts b/packages/xrpl/packages/ledgers/src/types.ts index 9ac2f8d..686818a 100644 --- a/packages/xrpl/packages/ledgers/src/types.ts +++ b/packages/xrpl/packages/ledgers/src/types.ts @@ -1,4 +1,4 @@ -import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; +import { ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; import { LedgerRequest } from "xrpl"; import { XrplLedgersIndexerEvents } from "./events"; @@ -11,6 +11,4 @@ export type SelectedLedgerType = Parameters; -export type XrplLedgersIndexerState = ExtendedIndexerState<{}>; - export type XrplLedgersIndexerIndexOptions = ExtendedIndexOptions<{}>; diff --git a/packages/xrpl/src/XrplIndexer.ts b/packages/xrpl/src/XrplIndexer.ts index c3fffcd..7eeebfb 100644 --- a/packages/xrpl/src/XrplIndexer.ts +++ b/packages/xrpl/src/XrplIndexer.ts @@ -6,7 +6,6 @@ export abstract class XrplIndexer extends provider: Generics["provider"] extends undefined ? XrplProvider : Generics["provider"]; events: Generics["events"]; config: Generics["config"]; - state: Generics["state"]; indexOptions: Generics["indexOptions"]; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -16,7 +15,7 @@ export abstract class XrplIndexer extends logger: { name: "XrplIndexer", }, - stateFilePath: "./.xrpl-indexer-state.json", + persistenceFilePath: "./.xrpl-indexer.json", } as typeof this.defaultConfig); } diff --git a/packages/xrpl/src/types.ts b/packages/xrpl/src/types.ts index 9281c82..8b0adbb 100644 --- a/packages/xrpl/src/types.ts +++ b/packages/xrpl/src/types.ts @@ -1,16 +1,13 @@ -import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; +import { ExtendedIndexOptions, ExtendedIndexerConfig } from "@bloxer/vanilla"; import { XrplProvider } from "./XrplProvider"; export type XrplIndexerConfig = ExtendedIndexerConfig<{}>; -export type XrplIndexerState = ExtendedIndexerState<{}>; - export type XrplIndexerIndexOptions = ExtendedIndexOptions<{}>; export type XrplIndexerGenerics = { provider?: XrplProvider; events: Record any>; config?: XrplIndexerConfig; - state?: XrplIndexerState; indexOptions?: XrplIndexerIndexOptions; }; diff --git a/scripts/create-flavour-impl.js b/scripts/create-flavour-impl.js index a9e2d12..2933596 100644 --- a/scripts/create-flavour-impl.js +++ b/scripts/create-flavour-impl.js @@ -18,7 +18,6 @@ function main() { implSignature, implIndexerName, implIndexerConfigName, - implIndexerStateName, implIndexOptionsName, implEventsName, } = buildFlavourImplementationNames(flavour, impl); @@ -103,12 +102,10 @@ Lightweight and simple ${flavour} ${impl} indexer based on custom events. fse.mkdirSync(buildImplPath("src")); // Create types - const implTypes = `import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "${vanillaPackage.name}" + const implTypes = `import { ExtendedIndexOptions, ExtendedIndexerConfig } from "${vanillaPackage.name}" export type ${implIndexerConfigName} = ExtendedIndexerConfig<{}>; -export type ${implIndexerStateName} = ExtendedIndexerState<{}>; - export type ${implIndexOptionsName} = ExtendedIndexOptions<{}>; `; fse.writeFileSync(buildImplSrcPath(`types.ts`), implTypes, "utf8"); @@ -121,13 +118,12 @@ export type ${implIndexOptionsName} = ExtendedIndexOptions<{}>; // Create FlavourImplIndexer class const implIndexer = `import { ${flavourIndexerName}, ${flavourProviderName} } from "${flavourPackage.name}"; import { ${implEventsName} } from "./events"; -import { ${implIndexerConfigName}, ${implIndexerStateName}, ${implIndexOptionsName} } from "./types"; +import { ${implIndexerConfigName}, ${implIndexOptionsName} } from "./types"; export class ${implIndexerName} extends ${flavourIndexerName}<{ provider: ${flavourProviderName}; events: ${implEventsName}; config: ${implIndexerConfigName}; - state: ${implIndexerStateName}; indexOptions: ${implIndexOptionsName}; }> { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -137,7 +133,7 @@ export class ${implIndexerName} extends ${flavourIndexerName}<{ logger: { name: "${implIndexerName}", }, - stateFilePath: "./.${implSignature}-indexer-state.json", + persistenceFilePath: "./.${implSignature}-indexer.db", } as typeof this.defaultConfig); } diff --git a/scripts/create-flavour.js b/scripts/create-flavour.js index 034cc9e..e361745 100644 --- a/scripts/create-flavour.js +++ b/scripts/create-flavour.js @@ -16,7 +16,6 @@ function main() { flavourIndexerName, flavourIndexerGenericsName, flavourExtendedIndexerConfigName, - flavourExtendedIndexerStateName, flavourExtendedIndexOptionsName, } = buildFlavourNames(flavour); @@ -121,20 +120,17 @@ export class ${flavourProviderName} extends Provider { fse.writeFileSync(buildFlavourSrcPath(`${flavourProviderName}.ts`), flavourProvider, "utf8"); // Create types - const flavourTypes = `import { ExtendedIndexerState, ExtendedIndexOptions, ExtendedIndexerConfig } from "${vanillaPackage.name}"; + const flavourTypes = `import { ExtendedIndexOptions, ExtendedIndexerConfig } from "${vanillaPackage.name}"; import { ${flavourProviderName} } from "./${flavourProviderName}"; export type ${flavourExtendedIndexerConfigName} = ExtendedIndexerConfig<{}>; -export type ${flavourExtendedIndexerStateName} = ExtendedIndexerState<{}>; - export type ${flavourExtendedIndexOptionsName} = ExtendedIndexOptions<{}>; export type ${flavourIndexerGenericsName} = { provider?: ${flavourProviderName}; events: Record any>; config?: ${flavourExtendedIndexerConfigName}; - state?: ${flavourExtendedIndexerStateName}; indexOptions?: ${flavourExtendedIndexOptionsName}; }; `; @@ -149,7 +145,6 @@ export abstract class ${flavourIndexerName} { protected overrideDefaultConfig(defaultConfig: typeof this.defaultConfig): void { @@ -159,7 +154,7 @@ export abstract class ${flavourIndexerName} Date: Thu, 7 Mar 2024 10:00:18 +0100 Subject: [PATCH 02/13] chore: remove unnecessary casts in `SQLBuilder` --- packages/vanilla/src/db/utils/SQLBuilder.ts | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/packages/vanilla/src/db/utils/SQLBuilder.ts b/packages/vanilla/src/db/utils/SQLBuilder.ts index 68dbc59..a8769fa 100644 --- a/packages/vanilla/src/db/utils/SQLBuilder.ts +++ b/packages/vanilla/src/db/utils/SQLBuilder.ts @@ -17,12 +17,7 @@ export class SQLBuilder { return `WHERE ${where .map((whereGroup) => - Object.entries( - this.entity.toRow( - // Casted to `InstanceOf`, `Entity.toRow` will have to handle possible undefined values - whereGroup as InstanceOf, - ), - ) + Object.entries(this.entity.toRow(whereGroup)) .map((key, value) => `${key} = ${value}`) .join(" AND "), ) @@ -72,12 +67,7 @@ export class SQLBuilder { buildSetClause(data: Partial>, ...where: Partial>[]): string { return this.withWhereClause( `UPDATE ${this.entity.table} - SET ${Object.entries( - this.entity.toRow( - // Casted to `InstanceOf`, `Entity.toRow` will have to handle possible undefined values - data as InstanceOf, - ), - ) + SET ${Object.entries(this.entity.toRow(data)) .map((key, value) => `${key} = ${value}`) .join(", ")}`, ...where, From d698a5bf478e1b87ad6995474c6264024cee8f01 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Thu, 7 Mar 2024 12:15:16 +0100 Subject: [PATCH 03/13] fix: add `event` to `LastEvent` and `PendingEvent` --- packages/vanilla/src/db/entities/LastEvent.ts | 1 + packages/vanilla/src/db/entities/PendingEvent.ts | 3 +++ packages/vanilla/src/db/migrations/001-init.sql | 12 ++++++++---- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/vanilla/src/db/entities/LastEvent.ts b/packages/vanilla/src/db/entities/LastEvent.ts index 116d947..1a72839 100644 --- a/packages/vanilla/src/db/entities/LastEvent.ts +++ b/packages/vanilla/src/db/entities/LastEvent.ts @@ -1,6 +1,7 @@ import { Entity } from "./Entity"; export class LastEvent extends Entity("last_event") { + event: string; hash: string; block: number; } diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts index 2067327..2fc1048 100644 --- a/packages/vanilla/src/db/entities/PendingEvent.ts +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -2,12 +2,14 @@ import { AnyObject } from "@swisstype/essential"; import { Entity } from "./Entity"; export class PendingEvent extends Entity("pending_event") { + event: string; hash: string; block: number; data: AnyObject; static fromRow(row: AnyObject): PendingEvent { return { + event: row.event, hash: row.hash, block: row.block, data: JSON.parse(row.data), @@ -16,6 +18,7 @@ export class PendingEvent extends Entity("pending_event") { static toRow(row: PendingEvent): AnyObject { return { + event: row.event, hash: row.hash, block: row.block, data: row.data ? JSON.stringify(row.data) : undefined, diff --git a/packages/vanilla/src/db/migrations/001-init.sql b/packages/vanilla/src/db/migrations/001-init.sql index 388a4d2..f592552 100644 --- a/packages/vanilla/src/db/migrations/001-init.sql +++ b/packages/vanilla/src/db/migrations/001-init.sql @@ -3,14 +3,18 @@ -------------------------------------------------------------------------------- CREATE TABLE pending_event ( - hash TEXT PRIMARY KEY, + event TEXT, + hash TEXT, block INTEGER NOT NULL, - data TEXT NOT NULL + data TEXT NOT NULL, + PRIMARY KEY (event, hash) ); CREATE TABLE last_event ( - hash TEXT PRIMARY KEY, - block INTEGER NOT NULL + event TEXT, + hash TEXT, + block INTEGER NOT NULL, + PRIMARY KEY (event, hash) ); -------------------------------------------------------------------------------- From eab8018a8851a21d5d7ad7bb98a8b0fc39a2ba3b Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Thu, 7 Mar 2024 18:39:26 +0100 Subject: [PATCH 04/13] feat: save pending events --- drills/ethers-typechain-contract/drill.ts | 2 +- .../src/EthersTypechainContractIndexer.ts | 16 ++----- packages/vanilla/src/Indexer.ts | 26 ++++++++-- .../vanilla/src/db/entities/PendingEvent.ts | 19 ++------ .../vanilla/src/db/migrations/001-init.sql | 2 +- .../src/db/repositories/SQLDB.repository.ts | 23 ++++----- .../src/db/repositories/SQLite.repository.ts | 5 +- .../adapters/{ => db}/SQLiteDB.adapter.ts | 0 .../repositories/adapters/{ => db}/index.ts | 0 .../{ => db}/interfaces/DB.adapter.ts | 0 .../adapters/{ => db}/interfaces/index.ts | 0 .../adapters/sql/SQL.adapter.ts} | 47 +++++++++++++++---- .../adapters/sql/SQLite.adapter.ts | 20 ++++++++ .../src/db/repositories/adapters/sql/index.ts | 2 + .../account/src/XrplAccountIndexer.ts | 18 +++---- .../ledgers/src/XrplLedgersIndexer.ts | 8 +--- 16 files changed, 117 insertions(+), 71 deletions(-) rename packages/vanilla/src/db/repositories/adapters/{ => db}/SQLiteDB.adapter.ts (100%) rename packages/vanilla/src/db/repositories/adapters/{ => db}/index.ts (100%) rename packages/vanilla/src/db/repositories/adapters/{ => db}/interfaces/DB.adapter.ts (100%) rename packages/vanilla/src/db/repositories/adapters/{ => db}/interfaces/index.ts (100%) rename packages/vanilla/src/db/{utils/SQLBuilder.ts => repositories/adapters/sql/SQL.adapter.ts} (63%) create mode 100644 packages/vanilla/src/db/repositories/adapters/sql/SQLite.adapter.ts create mode 100644 packages/vanilla/src/db/repositories/adapters/sql/index.ts diff --git a/drills/ethers-typechain-contract/drill.ts b/drills/ethers-typechain-contract/drill.ts index 27cbaae..4baa4b4 100644 --- a/drills/ethers-typechain-contract/drill.ts +++ b/drills/ethers-typechain-contract/drill.ts @@ -4,8 +4,8 @@ import { BridgeDoorCommon__factory } from "@peersyst/xrp-evm-contracts"; async function drill() { const indexer = new EthersTypechainContractIndexer("0x0FCCFB556B4aA1B44F31220AcDC8007D46514f31", BridgeDoorCommon__factory, { wsUrl: "ws://168.119.63.112:8546", - persist: false, persistenceFilePath: "persistence/.ethers-typechain-contract-indexer.db", + startingBlock: 4000000, }); await indexer.run(); } diff --git a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts index c35f208..e93b70e 100644 --- a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts +++ b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts @@ -54,6 +54,7 @@ export class EthersTypechainContractIndexer { let fromBlock = startingBlock; let toBlock = Math.min(startingBlock + this.config.blocksBatchSize, endingBlock); + // TODO: Move to `Indexer` in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s let reachedPreviousTransaction = !previousTransaction; while (fromBlock <= endingBlock) { @@ -79,24 +80,13 @@ export class EthersTypechainContractIndexer { private _defaultConfig: IndexerDefaultConfig = { @@ -331,6 +331,24 @@ export abstract class Indexer { } } + protected notifyEvent( + event: Event, + hash: string, + block: number, + ...data: Parameters + ): void { + // TODO: Check if the event is previous to the last event with a `reachedLastEvent` variable in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s + const pendingEventsRepository = this.db.getRepository(PendingEvent); + pendingEventsRepository + .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) + .then(() => { + this.emit(event, ...data); + }) + .catch((e) => { + this.logger.error(`Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`); + }); + } + /** * Runs the indexer */ @@ -338,14 +356,14 @@ export abstract class Indexer { await Promise.all([this.db.open(), this.initializeProvider()]); // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - let lastEvent = {} as any; + let lastEvent = undefined as any; let nextBlock: number | undefined; this.running = true; while (this.running) { this.latestBlock = await this.latestBlockPromise; - if (nextBlock !== undefined ? nextBlock <= this.latestBlock : !lastEvent.block || lastEvent.block <= this.latestBlock) { + if (nextBlock !== undefined ? nextBlock <= this.latestBlock : !lastEvent || lastEvent.block <= this.latestBlock) { const indexOptions = this.getIndexOptions(nextBlock, lastEvent); const indexingFrom = indexOptions.startingBlock ?? "genesis"; @@ -360,7 +378,7 @@ export abstract class Indexer { this.logger.info(`Retrying indexing from block ${indexingFrom} to latest...`); // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - lastEvent = {}; + lastEvent = undefined; nextBlock = undefined; } } diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts index 2fc1048..85a46fb 100644 --- a/packages/vanilla/src/db/entities/PendingEvent.ts +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -7,21 +7,12 @@ export class PendingEvent extends Entity("pending_event") { block: number; data: AnyObject; - static fromRow(row: AnyObject): PendingEvent { + static fromEventNotification(event: string, hash: string, block: number, ...data: AnyObject[]): PendingEvent { return { - event: row.event, - hash: row.hash, - block: row.block, - data: JSON.parse(row.data), - }; - } - - static toRow(row: PendingEvent): AnyObject { - return { - event: row.event, - hash: row.hash, - block: row.block, - data: row.data ? JSON.stringify(row.data) : undefined, + event, + hash, + block, + data: data.length ? data : undefined, }; } } diff --git a/packages/vanilla/src/db/migrations/001-init.sql b/packages/vanilla/src/db/migrations/001-init.sql index f592552..310ea73 100644 --- a/packages/vanilla/src/db/migrations/001-init.sql +++ b/packages/vanilla/src/db/migrations/001-init.sql @@ -6,7 +6,7 @@ CREATE TABLE pending_event ( event TEXT, hash TEXT, block INTEGER NOT NULL, - data TEXT NOT NULL, + data BLOB NOT NULL, PRIMARY KEY (event, hash) ); diff --git a/packages/vanilla/src/db/repositories/SQLDB.repository.ts b/packages/vanilla/src/db/repositories/SQLDB.repository.ts index 1aafd17..c93fa89 100644 --- a/packages/vanilla/src/db/repositories/SQLDB.repository.ts +++ b/packages/vanilla/src/db/repositories/SQLDB.repository.ts @@ -1,41 +1,38 @@ import { EntityConstructor } from "../entities"; import { InstanceOf } from "../../utils/types"; import { DBRepository } from "./DB.repository"; -import { SQLBuilder } from "../utils/SQLBuilder"; -import { DBAdapter } from "./adapters/interfaces"; +import { DBAdapter } from "./adapters/db/interfaces"; +import { SQLAdapter } from "./adapters/sql"; export abstract class SQLDBRepository extends DBRepository { - private readonly sqlBuilder: SQLBuilder; - constructor( protected readonly db: DBAdapter, entity: Entity, + protected readonly sqlAdapter: SQLAdapter, ) { super(entity); - - this.sqlBuilder = new SQLBuilder(entity); } async findOne(...where: Partial>[]): Promise> { - const row = await this.db.get(this.sqlBuilder.buildSelect(...where)); - return row ? this.entity.fromRow(row) : undefined; + const row = await this.db.get(this.sqlAdapter.buildSelect(...where)); + return row ? this.entity.fromRow(this.sqlAdapter.sqlRowToRecord(row)) : undefined; } async findAll(...where: Partial>[]): Promise[]> { - const rows = await this.db.all(this.sqlBuilder.buildSelect(...where)); - return rows.map((row) => this.entity.fromRow(row)); + const rows = await this.db.all(this.sqlAdapter.buildSelect(...where)); + return rows.map((row) => this.entity.fromRow(this.sqlAdapter.sqlRowToRecord(row))); } async create(data: InstanceOf): Promise> { - await this.db.create(this.sqlBuilder.buildInsert(data)); + await this.db.create(this.sqlAdapter.buildInsert(data)); return data; } async update(data: Partial>, ...where: Partial>[]): Promise { - await this.db.update(this.sqlBuilder.buildSetClause(data, ...where)); + await this.db.update(this.sqlAdapter.buildSetClause(data, ...where)); } async delete(...where: Partial>[]): Promise { - await this.db.delete(this.sqlBuilder.buildDelete(...where)); + await this.db.delete(this.sqlAdapter.buildDelete(...where)); } } diff --git a/packages/vanilla/src/db/repositories/SQLite.repository.ts b/packages/vanilla/src/db/repositories/SQLite.repository.ts index 7517a9a..bc05c30 100644 --- a/packages/vanilla/src/db/repositories/SQLite.repository.ts +++ b/packages/vanilla/src/db/repositories/SQLite.repository.ts @@ -1,10 +1,11 @@ import { Database } from "sqlite"; import { EntityConstructor } from "../entities"; import { SQLDBRepository } from "./SQLDB.repository"; -import { SQLiteDBAdapter } from "./adapters"; +import { SQLiteDBAdapter } from "./adapters/db"; +import { SQLiteAdapter } from "./adapters/sql"; export class SQLiteRepository extends SQLDBRepository { constructor(db: Database, entity: Entity) { - super(new SQLiteDBAdapter(db), entity); + super(new SQLiteDBAdapter(db), entity, new SQLiteAdapter(entity)); } } diff --git a/packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts b/packages/vanilla/src/db/repositories/adapters/db/SQLiteDB.adapter.ts similarity index 100% rename from packages/vanilla/src/db/repositories/adapters/SQLiteDB.adapter.ts rename to packages/vanilla/src/db/repositories/adapters/db/SQLiteDB.adapter.ts diff --git a/packages/vanilla/src/db/repositories/adapters/index.ts b/packages/vanilla/src/db/repositories/adapters/db/index.ts similarity index 100% rename from packages/vanilla/src/db/repositories/adapters/index.ts rename to packages/vanilla/src/db/repositories/adapters/db/index.ts diff --git a/packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts b/packages/vanilla/src/db/repositories/adapters/db/interfaces/DB.adapter.ts similarity index 100% rename from packages/vanilla/src/db/repositories/adapters/interfaces/DB.adapter.ts rename to packages/vanilla/src/db/repositories/adapters/db/interfaces/DB.adapter.ts diff --git a/packages/vanilla/src/db/repositories/adapters/interfaces/index.ts b/packages/vanilla/src/db/repositories/adapters/db/interfaces/index.ts similarity index 100% rename from packages/vanilla/src/db/repositories/adapters/interfaces/index.ts rename to packages/vanilla/src/db/repositories/adapters/db/interfaces/index.ts diff --git a/packages/vanilla/src/db/utils/SQLBuilder.ts b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts similarity index 63% rename from packages/vanilla/src/db/utils/SQLBuilder.ts rename to packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts index a8769fa..afcf884 100644 --- a/packages/vanilla/src/db/utils/SQLBuilder.ts +++ b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts @@ -1,12 +1,40 @@ -import { EntityConstructor } from "../entities"; -import { InstanceOf } from "../../utils/types"; +import { EntityConstructor } from "../../../entities"; +import { InstanceOf } from "../../../../utils/types"; +import { AnyObject } from "@swisstype/essential"; /** - * SQL builder for an entity. + * SQL adapter used to adapt JS values to SQL for given entity. */ -export class SQLBuilder { +export abstract class SQLAdapter { constructor(protected readonly entity: Entity) {} + /** + * Transforms a JS value to its SQL value. + * @param value The JS value. + * @returns The SQL value as string. + */ + abstract valueToSql(value: T): string; + + /** + * Transforms an SQL value to its JS value. + * @param value The SQL value. + * @returns The JS value. + */ + abstract sqlToValue(sqlValue: any): T; + + /** + * Transforms an SQL row to a JS record. + * @param row The SQL row. + * @returns The JS record. + */ + sqlRowToRecord(sqlRow: AnyObject): T { + const record = {}; + for (const [key, sqlValue] of Object.entries(sqlRow)) { + record[key] = this.sqlToValue(sqlValue); + } + return record as T; + } + /** * Builds a where clause. * @param where The where clauses (will be joined with OR). @@ -18,7 +46,7 @@ export class SQLBuilder { return `WHERE ${where .map((whereGroup) => Object.entries(this.entity.toRow(whereGroup)) - .map((key, value) => `${key} = ${value}`) + .map((key, value) => `${key} = ${this.valueToSql(value)}`) .join(" AND "), ) .join(" OR ")}`; @@ -54,8 +82,11 @@ export class SQLBuilder { * @returns The insert query. */ buildInsert(data: InstanceOf): string { - return `INSERT INTO ${this.entity.table} (${Object.keys(data).join(", ")}) - VALUES (${Object.values(this.entity.toRow(data)).join(", ")})`; + const rowData = this.entity.toRow(data); + return `INSERT INTO ${this.entity.table} (${Object.keys(rowData).join(", ")}) + VALUES (${Object.values(rowData) + .map((value) => this.valueToSql(value)) + .join(", ")})`; } /** @@ -68,7 +99,7 @@ export class SQLBuilder { return this.withWhereClause( `UPDATE ${this.entity.table} SET ${Object.entries(this.entity.toRow(data)) - .map((key, value) => `${key} = ${value}`) + .map((key, value) => `${key} = ${this.valueToSql(value)}`) .join(", ")}`, ...where, ); diff --git a/packages/vanilla/src/db/repositories/adapters/sql/SQLite.adapter.ts b/packages/vanilla/src/db/repositories/adapters/sql/SQLite.adapter.ts new file mode 100644 index 0000000..a85190b --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/sql/SQLite.adapter.ts @@ -0,0 +1,20 @@ +import { EntityConstructor } from "../../../entities"; +import { SQLAdapter } from "./SQL.adapter"; + +/** + * An SQLite adapter extended from the SQL adapter. + */ +export class SQLiteAdapter extends SQLAdapter { + valueToSql(value: T): string { + if (typeof value === "string") return `'${value}'`; + else if (value === undefined || value === null) return "NULL"; + else if (typeof value === "object") return `X'${Buffer.from(JSON.stringify(value), "utf-8").toString("hex")}'`; + else return value.toString(); + } + + sqlToValue(sqlValue: any): T { + if (typeof sqlValue === "string") return sqlValue as T; + else if (Buffer.isBuffer(sqlValue)) return JSON.parse(sqlValue.toString("utf-8")); + return sqlValue === null ? undefined : sqlValue; + } +} diff --git a/packages/vanilla/src/db/repositories/adapters/sql/index.ts b/packages/vanilla/src/db/repositories/adapters/sql/index.ts new file mode 100644 index 0000000..49ef99b --- /dev/null +++ b/packages/vanilla/src/db/repositories/adapters/sql/index.ts @@ -0,0 +1,2 @@ +export * from "./SQL.adapter"; +export * from "./SQLite.adapter"; diff --git a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts index 56dade4..45f8025 100644 --- a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts +++ b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts @@ -36,6 +36,7 @@ export class XrplAccountIndexer extends XrplIndexer<{ async index({ startingBlock, endingBlock, previousTransaction }: XrplAccountIndexerIndexOptions): Promise { let marker; let lastIndexedLedger = startingBlock; + // TODO: Move to `Indexer` in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s let reachedPreviousTransaction = !previousTransaction; do { @@ -63,15 +64,14 @@ export class XrplAccountIndexer extends XrplIndexer<{ // If the previous transaction has been reached the following ones can be indexed. Otherwise, it still needs to be found. if (reachedPreviousTransaction) { - // Emit transaction events - this.emit("Transaction", correctlyCastedAccountTx); - this.emit(tx.TransactionType, correctlyCastedAccountTx as AccountTransaction); - // Save the last indexed transaction state - // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=9d8b6213e44d4c1e8f2c805565bb2486&pm=s - // this.setPartialState({ - // transaction: tx.hash, - // block: tx.ledger_index, - // }); + const hash = tx.hash; + const block = tx.ledger_index; + + if (hash !== undefined && block !== undefined) { + // Notify transaction events + this.notifyEvent("Transaction", hash, block, correctlyCastedAccountTx); + this.notifyEvent(tx.TransactionType, hash, block, correctlyCastedAccountTx as AccountTransaction); + } } else if (tx.hash === previousTransaction) { reachedPreviousTransaction = true; } diff --git a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts index d35e70e..04199aa 100644 --- a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts +++ b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts @@ -35,15 +35,11 @@ export class XrplLedgersIndexer extends XrplIndexer<{ }); if (res.result.validated) { - this.emit("Ledger", res.result.ledger as SelectedLedgerType); + this.notifyEvent("Ledger", res.result.ledger_hash, res.result.ledger_index, res.result.ledger as SelectedLedgerType); ++currentBlock; } - // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=9d8b6213e44d4c1e8f2c805565bb2486&pm=s - // this.setState({ - // hash: res.result.ledger.ledger_hash - // block: res.result.ledger.ledger_index, - // } as InheritedIndexerState); + // TODO: Should we add a delay here? } const nextLedger = currentBlock; From ccf20b621ffabe6f83e67adde30297f7780d819b Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Fri, 8 Mar 2024 10:01:32 +0100 Subject: [PATCH 05/13] fix: don't open/close `Indexer.db` if `persist` is false --- packages/vanilla/src/Indexer.ts | 55 ++++++++++++++++++------ packages/vanilla/src/db/interfaces/DB.ts | 1 + 2 files changed, 44 insertions(+), 12 deletions(-) diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index 131017f..16a1cab 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -316,6 +316,24 @@ export abstract class Indexer { } } + /** + * Opens the database connection if `persist` is true. + */ + private async openDB(): Promise { + if (this.config.persist) { + await this.db.open(); + } + } + + /** + * Closes the database connection if `persist` is true. + */ + private async closeDB(): Promise { + if (this.config.persist) { + await this.db.close(); + } + } + /** * Gets index options. * @param nextBlock The next block. @@ -331,29 +349,42 @@ export abstract class Indexer { } } + /** + * Creates a pending event if `persist` is true and emits the event. + * @param event The event to emit. + * @param hash The hash of the event. + * @param block The block of the event. + * @param data The data of the event. + */ protected notifyEvent( event: Event, hash: string, block: number, ...data: Parameters ): void { - // TODO: Check if the event is previous to the last event with a `reachedLastEvent` variable in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - const pendingEventsRepository = this.db.getRepository(PendingEvent); - pendingEventsRepository - .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) - .then(() => { - this.emit(event, ...data); - }) - .catch((e) => { - this.logger.error(`Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`); - }); + if (this.config.persist) { + // TODO: Check if the event is previous to the last event with a `reachedLastEvent` variable in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s + const pendingEventsRepository = this.db.getRepository(PendingEvent); + pendingEventsRepository + .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) + .then(() => { + this.emit(event, ...data); + }) + .catch((e) => { + this.logger.error( + `Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, + ); + }); + } else { + this.emit(event, ...data); + } } /** * Runs the indexer */ async run(): Promise { - await Promise.all([this.db.open(), this.initializeProvider()]); + await Promise.all([this.openDB(), this.initializeProvider()]); // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s let lastEvent = undefined as any; @@ -393,7 +424,7 @@ export abstract class Indexer { this.logger.info("Stopping indexer..."); this.running = false; - await Promise.all([this.db.close(), this.unsubscribeFromLatestBlock()]); + await Promise.all([this.closeDB(), this.unsubscribeFromLatestBlock()]); this.logger.info("Indexer stopped"); } else { diff --git a/packages/vanilla/src/db/interfaces/DB.ts b/packages/vanilla/src/db/interfaces/DB.ts index 7d3b3d5..4a791ca 100644 --- a/packages/vanilla/src/db/interfaces/DB.ts +++ b/packages/vanilla/src/db/interfaces/DB.ts @@ -4,6 +4,7 @@ import { EntityConstructor } from "../entities"; export interface DB { /** * Opens the database connection. + * The database will be created if it does not exist. */ open(): Promise; From d61283412a867231d899c8e1ca96c269ca7500d9 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Fri, 8 Mar 2024 12:29:27 +0100 Subject: [PATCH 06/13] feat: recover pending events and last event --- .../src/EthersTypechainContractIndexer.ts | 18 +- packages/vanilla/src/Indexer.ts | 165 +++++++++++++++--- .../vanilla/src/db/entities/PendingEvent.ts | 5 +- .../src/db/repositories/DB.repository.ts | 2 +- .../src/db/repositories/SQLDB.repository.ts | 2 +- packages/vanilla/src/types.ts | 12 +- .../account/src/XrplAccountIndexer.ts | 21 +-- .../ledgers/src/XrplLedgersIndexer.ts | 2 +- scripts/create-flavour-impl.js | 2 +- 9 files changed, 160 insertions(+), 69 deletions(-) diff --git a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts index e93b70e..517a6d9 100644 --- a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts +++ b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts @@ -47,15 +47,9 @@ export class EthersTypechainContractIndexer { + async index({ startingBlock, endingBlock }: EthersTypechainContractIndexerIndexOptions): Promise { let fromBlock = startingBlock; let toBlock = Math.min(startingBlock + this.config.blocksBatchSize, endingBlock); - // TODO: Move to `Indexer` in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - let reachedPreviousTransaction = !previousTransaction; while (fromBlock <= endingBlock) { this.logger.info(`Indexing from block ${fromBlock} to block ${toBlock}...`); @@ -77,14 +71,8 @@ export class EthersTypechainContractIndexer { private _defaultConfig: IndexerDefaultConfig = { startingBlock: 0, + endingBlock: "latest", reconnectTimeout: 5000, maxReconnectAttempts: 10, maxRequestRetries: 10, @@ -80,22 +81,35 @@ export abstract class Indexer { * The number of reconnect retries */ private reconnectRetries = 0; + /** * Provider (connection) promises */ private resolveProviderPromise: () => void; private providerPromise: Promise; + /** * Block promises */ private resolveLatestBlockPromise: (block: number) => void; private latestBlockPromise: Promise; protected latestBlock: number; + /** * Whether the indexer is running */ running = false; + /** + * The last event processed before the indexer is started. + */ + lastEvent: LastEvent | undefined; + + /** + * Whether the last event has been reached and new events can be processed. + */ + reachedLastEvent = false; + /** * Event listener for the indexer */ @@ -334,18 +348,58 @@ export abstract class Indexer { } } + /** + * Gets the starting block. + * @returns The starting block. + */ + private getStartingBlock(): number { + if (this.lastEvent) { + return this.lastEvent.block; + } else { + return this.config.startingBlock === "latest" ? this.latestBlock : this.config.startingBlock; + } + } + + /** + * Gets the ending block. + * @returns The ending block. + */ + private getEndingBlock(): number { + return this.config.endingBlock === "latest" ? this.latestBlock : this.config.endingBlock; + } + /** * Gets index options. * @param nextBlock The next block. * @param lastEvent The last event. + * @returns The index options. + */ + private getIndexOptions(nextBlock: number | undefined): Required { + return { + startingBlock: nextBlock ?? this.getStartingBlock(), + endingBlock: this.getEndingBlock(), + }; + } + + /** + * Checks if indexer can index. + * The indexer can index if the staring block is less than or equal to the latest block. + * @param indexOptions The index options. + * @returns Whether the indexer can index. */ - private getIndexOptions(nextBlock: number | undefined, lastEvent: LastEvent | undefined): IndexOptions { - if (nextBlock) { - return { startingBlock: nextBlock }; - } else if (lastEvent) { - return { startingBlock: lastEvent.block, previousTransaction: lastEvent.hash }; + private async canIndex({ startingBlock, endingBlock }: Required): Promise { + if (this.lastEvent && this.lastEvent.block > this.latestBlock) { + this.logger.fatal( + `Last event block ${this.lastEvent.block} is greater than the latest block ${this.latestBlock}. This should never happen and could mean that the database data has not been stored correctly or is corrupted.`, + ); + await this.stop(); + return false; + } else if (startingBlock > endingBlock) { + this.logger.info(`Indexing to block ${endingBlock} completed.`); + await this.stop(); + return false; } else { - return { startingBlock: this.config.startingBlock === "latest" ? this.latestBlock : this.config.startingBlock }; + return true; } } @@ -363,39 +417,97 @@ export abstract class Indexer { ...data: Parameters ): void { if (this.config.persist) { - // TODO: Check if the event is previous to the last event with a `reachedLastEvent` variable in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - const pendingEventsRepository = this.db.getRepository(PendingEvent); - pendingEventsRepository - .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) - .then(() => { - this.emit(event, ...data); - }) - .catch((e) => { - this.logger.error( - `Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, - ); - }); + if (this.reachedLastEvent) { + this.db + .getRepository(PendingEvent) + .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) + .then(() => { + this.emit(event, ...data); + }) + .catch((e) => { + this.logger.error( + `Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, + ); + }); + } else { + // We can assert `this.lastEvent` since it has to be defined for `this.reachedLastEvent` to be false. + if (this.lastEvent!.event === event && this.lastEvent!.hash === hash) { + this.reachedLastEvent = true; + } + } } else { this.emit(event, ...data); } } + /** + * Gets the last event processed. + * @returns The last event processed or undefined if there is no last event. + */ + private getLastEvent(): Promise { + return this.db.getRepository(LastEvent).findOne(); + } + + /** + * Gets the pending events. + * @returns The pending events. + */ + private getPendingEvents(): Promise { + return this.db.getRepository(PendingEvent).findAll(); + } + + /** + * Recovers the last event processed. + * Sets `lastEvent` and `reachedLastEvent`. + */ + private async recoverLastEvent(): Promise { + if (this.config.persist) { + this.lastEvent = await this.getLastEvent(); + this.reachedLastEvent = !this.lastEvent; + } else { + this.lastEvent = undefined; + this.reachedLastEvent = true; + } + } + + /** + * Recovers the pending events and emits them. + */ + private async recoverPendingEvents(): Promise { + const pendingEvents = await this.getPendingEvents(); + + for (const pendingEvent of pendingEvents) { + this.emit(pendingEvent.event, ...(pendingEvent.data as Parameters)); + } + } + /** * Runs the indexer */ async run(): Promise { - await Promise.all([this.openDB(), this.initializeProvider()]); + this.running = true; + + try { + // Open db connection and initialize provider + await Promise.all([this.openDB(), this.initializeProvider()]); + // Recover the last event + await this.recoverLastEvent(); + // Recover the pending events + await this.recoverPendingEvents(); + } catch (e) { + this.logger.error(`Error while running the indexer: ${e}`); + await this.stop(); + } - // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - let lastEvent = undefined as any; let nextBlock: number | undefined; - this.running = true; while (this.running) { this.latestBlock = await this.latestBlockPromise; - if (nextBlock !== undefined ? nextBlock <= this.latestBlock : !lastEvent || lastEvent.block <= this.latestBlock) { - const indexOptions = this.getIndexOptions(nextBlock, lastEvent); + const indexOptions = this.getIndexOptions(nextBlock); + const canIndex = await this.canIndex(indexOptions); + + if (canIndex) { const indexingFrom = indexOptions.startingBlock ?? "genesis"; this.logger.info(`Indexing from block ${indexingFrom} to latest`); @@ -407,10 +519,6 @@ export abstract class Indexer { } catch (e) { this.logger.error(`Indexing from block ${indexingFrom} to latest failed with error ${e}`); this.logger.info(`Retrying indexing from block ${indexingFrom} to latest...`); - - // TODO: Implement with db in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - lastEvent = undefined; - nextBlock = undefined; } } } @@ -424,6 +532,7 @@ export abstract class Indexer { this.logger.info("Stopping indexer..."); this.running = false; + // Close db connection and unsubscribe from latest block await Promise.all([this.closeDB(), this.unsubscribeFromLatestBlock()]); this.logger.info("Indexer stopped"); diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts index 85a46fb..8924def 100644 --- a/packages/vanilla/src/db/entities/PendingEvent.ts +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -1,13 +1,12 @@ -import { AnyObject } from "@swisstype/essential"; import { Entity } from "./Entity"; export class PendingEvent extends Entity("pending_event") { event: string; hash: string; block: number; - data: AnyObject; + data: any[]; - static fromEventNotification(event: string, hash: string, block: number, ...data: AnyObject[]): PendingEvent { + static fromEventNotification(event: string, hash: string, block: number, ...data: any[]): PendingEvent { return { event, hash, diff --git a/packages/vanilla/src/db/repositories/DB.repository.ts b/packages/vanilla/src/db/repositories/DB.repository.ts index 1f57009..33a916e 100644 --- a/packages/vanilla/src/db/repositories/DB.repository.ts +++ b/packages/vanilla/src/db/repositories/DB.repository.ts @@ -9,7 +9,7 @@ export abstract class DBRepository { * @param where The where clauses (will be joined with OR). * @returns The resource or undefined if no resource is found. */ - abstract findOne(...where: Partial>[]): Promise>; + abstract findOne(...where: Partial>[]): Promise | undefined>; /** * Returns all resources from the DB matching the given wheres. diff --git a/packages/vanilla/src/db/repositories/SQLDB.repository.ts b/packages/vanilla/src/db/repositories/SQLDB.repository.ts index c93fa89..e2c4749 100644 --- a/packages/vanilla/src/db/repositories/SQLDB.repository.ts +++ b/packages/vanilla/src/db/repositories/SQLDB.repository.ts @@ -13,7 +13,7 @@ export abstract class SQLDBRepository extends super(entity); } - async findOne(...where: Partial>[]): Promise> { + async findOne(...where: Partial>[]): Promise | undefined> { const row = await this.db.get(this.sqlAdapter.buildSelect(...where)); return row ? this.entity.fromRow(this.sqlAdapter.sqlRowToRecord(row)) : undefined; } diff --git a/packages/vanilla/src/types.ts b/packages/vanilla/src/types.ts index 5e423b8..97ca01f 100644 --- a/packages/vanilla/src/types.ts +++ b/packages/vanilla/src/types.ts @@ -10,8 +10,14 @@ export type IndexerConfig = { wsUrl: string; /** * The starting block to index from. - * */ + * @default 0 + */ startingBlock?: number | undefined | "latest"; + /** + * The ending block to index to. + * @default "latest" + */ + endingBlock?: number | undefined | "latest"; /** * The maximum interval in milliseconds to send a ping request to the node. * @default 5000 @@ -61,10 +67,6 @@ export type IndexOptions = { * @default "The current block." */ endingBlock?: number; - /** - * The hash of the previous transaction. - */ - previousTransaction?: string; }; export type ExtendedIndexOptions = (ExtendedOptions extends undefined diff --git a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts index 45f8025..a0e8059 100644 --- a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts +++ b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts @@ -33,11 +33,9 @@ export class XrplAccountIndexer extends XrplIndexer<{ super(config); } - async index({ startingBlock, endingBlock, previousTransaction }: XrplAccountIndexerIndexOptions): Promise { + async index({ startingBlock, endingBlock }: XrplAccountIndexerIndexOptions): Promise { let marker; let lastIndexedLedger = startingBlock; - // TODO: Move to `Indexer` in https://www.notion.so/1930f38fb1e94f82845dab04ac1caeca?v=64f1d5da841741cf9cb3b831e5b493e3&p=fbd10cc7c64f44deb868638b7ac89c86&pm=s - let reachedPreviousTransaction = !previousTransaction; do { const res = await this.request({ @@ -62,18 +60,13 @@ export class XrplAccountIndexer extends XrplIndexer<{ if (accountTx.validated && correctlyCastedAccountTx.meta.TransactionResult === "tesSUCCESS") { const tx = correctlyCastedAccountTx.tx; - // If the previous transaction has been reached the following ones can be indexed. Otherwise, it still needs to be found. - if (reachedPreviousTransaction) { - const hash = tx.hash; - const block = tx.ledger_index; + const hash = tx.hash; + const block = tx.ledger_index; - if (hash !== undefined && block !== undefined) { - // Notify transaction events - this.notifyEvent("Transaction", hash, block, correctlyCastedAccountTx); - this.notifyEvent(tx.TransactionType, hash, block, correctlyCastedAccountTx as AccountTransaction); - } - } else if (tx.hash === previousTransaction) { - reachedPreviousTransaction = true; + if (hash !== undefined && block !== undefined) { + // Notify transaction events + this.notifyEvent("Transaction", hash, block, correctlyCastedAccountTx); + this.notifyEvent(tx.TransactionType, hash, block, correctlyCastedAccountTx as AccountTransaction); } } } diff --git a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts index 04199aa..18a5489 100644 --- a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts +++ b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts @@ -23,7 +23,7 @@ export class XrplLedgersIndexer extends XrplIndexer<{ super(config); } - async index({ startingBlock, endingBlock = this.latestBlock }: XrplLedgersIndexerIndexOptions): Promise { + async index({ startingBlock, endingBlock }: XrplLedgersIndexerIndexOptions): Promise { let currentBlock = startingBlock; while (currentBlock <= endingBlock) { diff --git a/scripts/create-flavour-impl.js b/scripts/create-flavour-impl.js index 2933596..5bea029 100644 --- a/scripts/create-flavour-impl.js +++ b/scripts/create-flavour-impl.js @@ -141,7 +141,7 @@ export class ${implIndexerName} extends ${flavourIndexerName}<{ super(config); } - async index({ startingBlock, endingBlock, previousTransaction }: ${implIndexOptionsName}): Promise {} + async index({ startingBlock, endingBlock }: ${implIndexOptionsName}): Promise {} } `; fse.writeFileSync(buildImplSrcPath(`${implIndexerName}.ts`), implIndexer, "utf8"); From 55ec65526e6110b260aa83cf8489cfe0119eb11f Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 12:19:29 +0100 Subject: [PATCH 07/13] feat: store last event --- package-lock.json | 12 +++- .../src/EthersTypechainContractIndexer.ts | 2 + packages/vanilla/package.json | 1 + packages/vanilla/src/Indexer.ts | 69 +++++++++++++++++-- packages/vanilla/src/db/entities/Entity.ts | 2 +- packages/vanilla/src/db/entities/LastEvent.ts | 17 ++++- .../vanilla/src/db/entities/PendingEvent.ts | 16 +++-- .../vanilla/src/db/migrations/001-init.sql | 3 +- .../src/db/repositories/DB.repository.ts | 8 +++ .../src/db/repositories/SQLDB.repository.ts | 10 +++ .../repositories/adapters/sql/SQL.adapter.ts | 2 +- .../account/src/XrplAccountIndexer.ts | 2 + .../ledgers/src/XrplLedgersIndexer.ts | 1 + 13 files changed, 125 insertions(+), 20 deletions(-) diff --git a/package-lock.json b/package-lock.json index b82badc..9a5a1b9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2710,6 +2710,14 @@ "integrity": "sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ==", "dev": true }, + "node_modules/async-mutex": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.4.1.tgz", + "integrity": "sha512-WfoBo4E/TbCX1G95XTjbWTE3X2XLG0m1Xbv2cwOtuPdyH9CZvnaA5nCt1ucjaKEgW2A5IF71hxrRhr83Je5xjA==", + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -10793,8 +10801,7 @@ "node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==", - "dev": true + "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" }, "node_modules/tslog": { "version": "4.9.2", @@ -11490,6 +11497,7 @@ "version": "0.0.2", "license": "ISC", "dependencies": { + "async-mutex": "^0.4.1", "fs-extra": "^11.2.0", "sqlite": "^5.1.1", "sqlite3": "^5.1.7", diff --git a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts index 517a6d9..28c00ae 100644 --- a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts +++ b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts @@ -75,6 +75,8 @@ export class EthersTypechainContractIndexer { private _defaultConfig: IndexerDefaultConfig = { @@ -77,6 +78,12 @@ export abstract class Indexer { */ private readonly db: DB; + /** + * A mutex used to ensure that only one new pending event is saved at a time. + * There could be a race condition where last event 2 is saved before last event 1. + */ + private readonly eventMutex = new Mutex(); + /** * The number of reconnect retries */ @@ -403,6 +410,42 @@ export abstract class Indexer { } } + /** + * Saves a new pending event by saving the last event and creating a new pending event. + * Uses a mutex to ensure that only one pending event is saved at a time. + * @param event The event to emit. + * @param hash The hash of the event. + * @param block The block of the event. + * @param data The data of the event. + * @returns An array containing the last event and the pending event. + */ + private async saveNewPendingEvent( + event: Event, + hash: string, + block: number, + ...data: Parameters + ): Promise<[LastEvent | void, PendingEvent]> { + return await this.eventMutex.runExclusive( + async () => + await Promise.all([ + this.db.getRepository(LastEvent).updateOrCreate(new LastEvent(block, event as string, hash)), + this.db.getRepository(PendingEvent).create(new PendingEvent(event as string, hash, block, ...data)), + ]), + ); + } + + /** + * Saves the next block to index by saving the last event only with the block. + * Uses a mutex to ensure that only one event is saved at a time. + * @param block The block. + * @returns An array containing the last event and the pending event. + */ + private async saveNextBlock(block: number): Promise { + return await this.eventMutex.runExclusive( + async () => await this.db.getRepository(LastEvent).updateOrCreate(new LastEvent(block + 1)), + ); + } + /** * Creates a pending event if `persist` is true and emits the event. * @param event The event to emit. @@ -418,15 +461,13 @@ export abstract class Indexer { ): void { if (this.config.persist) { if (this.reachedLastEvent) { - this.db - .getRepository(PendingEvent) - .create(PendingEvent.fromEventNotification(event as string, hash, block, ...data)) + this.saveNewPendingEvent(event, hash, block, ...data) .then(() => { this.emit(event, ...data); }) .catch((e) => { this.logger.error( - `Error while creating the pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, + `Error while saving new pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, ); }); } else { @@ -440,6 +481,23 @@ export abstract class Indexer { } } + /** + * Notifies the last block processed. It saves the last event only with the block if `persist` is true. + * This method should be called after processing a block or a batch of blocks in order to keep the persistence updated. + * @param block The block. + */ + protected notifyBlock(block: number): void { + if (this.config.persist) { + /** + * We assume that the last event is reached. + * A new block can never be processed if the last event is not reached. + */ + this.saveNextBlock(block).catch((e) => { + this.logger.error(`Error while saving the last block processed ${block}: ${e}`); + }); + } + } + /** * Gets the last event processed. * @returns The last event processed or undefined if there is no last event. @@ -463,7 +521,8 @@ export abstract class Indexer { private async recoverLastEvent(): Promise { if (this.config.persist) { this.lastEvent = await this.getLastEvent(); - this.reachedLastEvent = !this.lastEvent; + // If last event is undefined OR it has no event (only contains the last block), last event is reached + this.reachedLastEvent = !this.lastEvent?.event; } else { this.lastEvent = undefined; this.reachedLastEvent = true; diff --git a/packages/vanilla/src/db/entities/Entity.ts b/packages/vanilla/src/db/entities/Entity.ts index 42c3524..85cf9c5 100644 --- a/packages/vanilla/src/db/entities/Entity.ts +++ b/packages/vanilla/src/db/entities/Entity.ts @@ -3,7 +3,7 @@ import { AnyObject } from "@swisstype/essential"; export interface EntityConstructor { // The entity type is set to `any` since the entity class is not known at compile time. // A constructor function must be defined in the interface so TS understands that another class can extend it. - new (): any; + new (...args: any[]): any; table: string; fromRow(row: AnyObject): any; toRow(entity: any): AnyObject; diff --git a/packages/vanilla/src/db/entities/LastEvent.ts b/packages/vanilla/src/db/entities/LastEvent.ts index 1a72839..7eaafb3 100644 --- a/packages/vanilla/src/db/entities/LastEvent.ts +++ b/packages/vanilla/src/db/entities/LastEvent.ts @@ -1,7 +1,20 @@ import { Entity } from "./Entity"; +/** + * Entity used to store the last event processed. + * There are 2 types of `LastEvent`: + * - Only `block`: Stores the upcoming block to be processed. + * - `block`, `event`, `hash`: Stores the last event processed. The indexer will start indexing from the next event. + */ export class LastEvent extends Entity("last_event") { - event: string; - hash: string; block: number; + event?: string; + hash?: string; + + constructor(block: number, event?: string, hash?: string) { + super(); + this.event = event; + this.hash = hash; + this.block = block; + } } diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts index 8924def..60bb108 100644 --- a/packages/vanilla/src/db/entities/PendingEvent.ts +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -1,17 +1,19 @@ import { Entity } from "./Entity"; +/** + * Entity used to store the pending events to be processed. + */ export class PendingEvent extends Entity("pending_event") { event: string; hash: string; block: number; data: any[]; - static fromEventNotification(event: string, hash: string, block: number, ...data: any[]): PendingEvent { - return { - event, - hash, - block, - data: data.length ? data : undefined, - }; + constructor(event: string, hash: string, block: number, ...data: any[]) { + super(); + this.event = event; + this.hash = hash; + this.block = block; + this.data = data; } } diff --git a/packages/vanilla/src/db/migrations/001-init.sql b/packages/vanilla/src/db/migrations/001-init.sql index 310ea73..424a57a 100644 --- a/packages/vanilla/src/db/migrations/001-init.sql +++ b/packages/vanilla/src/db/migrations/001-init.sql @@ -13,8 +13,7 @@ CREATE TABLE pending_event ( CREATE TABLE last_event ( event TEXT, hash TEXT, - block INTEGER NOT NULL, - PRIMARY KEY (event, hash) + block INTEGER NOT NULL ); -------------------------------------------------------------------------------- diff --git a/packages/vanilla/src/db/repositories/DB.repository.ts b/packages/vanilla/src/db/repositories/DB.repository.ts index 33a916e..a07710f 100644 --- a/packages/vanilla/src/db/repositories/DB.repository.ts +++ b/packages/vanilla/src/db/repositories/DB.repository.ts @@ -37,4 +37,12 @@ export abstract class DBRepository { * @param where The where clauses (will be joined with OR). */ abstract delete(...where: Partial>[]): Promise; + + /** + * Updates a resource in the DB or creates it if it does not exist. + * @param data The data to update or create. + * @param where The where clauses (will be joined with OR). + * @returns The created resource if it was created, otherwise void. + */ + abstract updateOrCreate(data: InstanceOf, ...where: Partial>[]): Promise | void>; } diff --git a/packages/vanilla/src/db/repositories/SQLDB.repository.ts b/packages/vanilla/src/db/repositories/SQLDB.repository.ts index e2c4749..56b9e84 100644 --- a/packages/vanilla/src/db/repositories/SQLDB.repository.ts +++ b/packages/vanilla/src/db/repositories/SQLDB.repository.ts @@ -35,4 +35,14 @@ export abstract class SQLDBRepository extends async delete(...where: Partial>[]): Promise { await this.db.delete(this.sqlAdapter.buildDelete(...where)); } + + async updateOrCreate(data: InstanceOf, ...where: Partial>[]): Promise | void> { + const row = await this.findOne(...where); + + if (row) { + return this.update(data, ...where); + } else { + return this.create(data); + } + } } diff --git a/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts index afcf884..be5634a 100644 --- a/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts +++ b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts @@ -99,7 +99,7 @@ export abstract class SQLAdapter { return this.withWhereClause( `UPDATE ${this.entity.table} SET ${Object.entries(this.entity.toRow(data)) - .map((key, value) => `${key} = ${this.valueToSql(value)}`) + .map(([key, value]) => `${key} = ${this.valueToSql(value)}`) .join(", ")}`, ...where, ); diff --git a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts index a0e8059..daff31b 100644 --- a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts +++ b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts @@ -70,6 +70,8 @@ export class XrplAccountIndexer extends XrplIndexer<{ } } } + + this.notifyBlock(lastIndexedLedger); } } while (marker); diff --git a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts index 18a5489..b427a7d 100644 --- a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts +++ b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts @@ -36,6 +36,7 @@ export class XrplLedgersIndexer extends XrplIndexer<{ if (res.result.validated) { this.notifyEvent("Ledger", res.result.ledger_hash, res.result.ledger_index, res.result.ledger as SelectedLedgerType); + this.notifyBlock(res.result.ledger_index); ++currentBlock; } From e3f6d0ef6fe23ed45e1500c1321683f2848030b8 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 12:25:42 +0100 Subject: [PATCH 08/13] refactor: change `Indexer.saveNextBlock` to `Indexer.saveBlock` --- packages/vanilla/src/Indexer.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index cc53af0..f55f639 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -435,12 +435,12 @@ export abstract class Indexer { } /** - * Saves the next block to index by saving the last event only with the block. + * Saves the last indexed block by saving the last event only with the block + 1. * Uses a mutex to ensure that only one event is saved at a time. * @param block The block. * @returns An array containing the last event and the pending event. */ - private async saveNextBlock(block: number): Promise { + private async saveBlock(block: number): Promise { return await this.eventMutex.runExclusive( async () => await this.db.getRepository(LastEvent).updateOrCreate(new LastEvent(block + 1)), ); @@ -492,7 +492,7 @@ export abstract class Indexer { * We assume that the last event is reached. * A new block can never be processed if the last event is not reached. */ - this.saveNextBlock(block).catch((e) => { + this.saveBlock(block).catch((e) => { this.logger.error(`Error while saving the last block processed ${block}: ${e}`); }); } From dc6cddac2f6a8b250663d796a0650097780759cb Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 12:27:05 +0100 Subject: [PATCH 09/13] chore: update `Indexer.saveBlock` and `Indexer.notifyBlock` headers --- packages/vanilla/src/Indexer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index f55f639..c6c255f 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -482,7 +482,7 @@ export abstract class Indexer { } /** - * Notifies the last block processed. It saves the last event only with the block if `persist` is true. + * Notifies the last block processed. It saves the last event only with the `block + 1` if `persist` is true. * This method should be called after processing a block or a batch of blocks in order to keep the persistence updated. * @param block The block. */ From 4d5b9496044ef33830689fc83701a177ca4add41 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 12:27:50 +0100 Subject: [PATCH 10/13] chore: change `LastEvent` constructor parameters order --- packages/vanilla/src/Indexer.ts | 2 +- packages/vanilla/src/db/entities/LastEvent.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index c6c255f..41dfb9f 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -435,7 +435,7 @@ export abstract class Indexer { } /** - * Saves the last indexed block by saving the last event only with the block + 1. + * Saves the last indexed block by saving the last event only with the `block + 1`. * Uses a mutex to ensure that only one event is saved at a time. * @param block The block. * @returns An array containing the last event and the pending event. diff --git a/packages/vanilla/src/db/entities/LastEvent.ts b/packages/vanilla/src/db/entities/LastEvent.ts index 7eaafb3..172dc79 100644 --- a/packages/vanilla/src/db/entities/LastEvent.ts +++ b/packages/vanilla/src/db/entities/LastEvent.ts @@ -13,8 +13,8 @@ export class LastEvent extends Entity("last_event") { constructor(block: number, event?: string, hash?: string) { super(); + this.block = block; this.event = event; this.hash = hash; - this.block = block; } } From 8550120b8031b24b8322bd7f033c2f7f2b0330d6 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 13:26:57 +0100 Subject: [PATCH 11/13] feat: add `Indexer.eventDone` --- drills/ethers-typechain-contract/drill.ts | 7 ++++++- packages/vanilla/src/Indexer.ts | 15 +++++++++++++++ .../db/repositories/adapters/sql/SQL.adapter.ts | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/drills/ethers-typechain-contract/drill.ts b/drills/ethers-typechain-contract/drill.ts index 4baa4b4..e24a779 100644 --- a/drills/ethers-typechain-contract/drill.ts +++ b/drills/ethers-typechain-contract/drill.ts @@ -2,11 +2,16 @@ import { EthersTypechainContractIndexer } from "@bloxer/ethers-typechain-contrac import { BridgeDoorCommon__factory } from "@peersyst/xrp-evm-contracts"; async function drill() { - const indexer = new EthersTypechainContractIndexer("0x0FCCFB556B4aA1B44F31220AcDC8007D46514f31", BridgeDoorCommon__factory, { + const indexer = new EthersTypechainContractIndexer("0xa6262F3c85617a1Bd17d5a82153A2c6e69AEcfeD", BridgeDoorCommon__factory, { wsUrl: "ws://168.119.63.112:8546", persistenceFilePath: "persistence/.ethers-typechain-contract-indexer.db", startingBlock: 4000000, }); + indexer.on("CreateAccount", (event) => { + setTimeout(() => { + indexer.eventDone("CreateAccount", event.transactionHash, event.blockNumber); + }, 5000); + }); await indexer.run(); } diff --git a/packages/vanilla/src/Indexer.ts b/packages/vanilla/src/Indexer.ts index 41dfb9f..a024bdc 100644 --- a/packages/vanilla/src/Indexer.ts +++ b/packages/vanilla/src/Indexer.ts @@ -540,6 +540,21 @@ export abstract class Indexer { } } + /** + * Marks an event as done by deleting the pending event. + * **This method should be called after processing an event.** + * If not called, the event will persist in the database and will be processed again after a restart. + * Does nothing when called with `persist` set to `false`. + * @param event The event. + * @param hash The hash of the event. + * @param block The block of the event. + */ + eventDone(event: Event, hash: string, block: number): void { + if (this.config.persist) { + this.db.getRepository(PendingEvent).delete({ event: event as string, hash, block }); + } + } + /** * Runs the indexer */ diff --git a/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts index be5634a..cb6107f 100644 --- a/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts +++ b/packages/vanilla/src/db/repositories/adapters/sql/SQL.adapter.ts @@ -46,7 +46,7 @@ export abstract class SQLAdapter { return `WHERE ${where .map((whereGroup) => Object.entries(this.entity.toRow(whereGroup)) - .map((key, value) => `${key} = ${this.valueToSql(value)}`) + .map(([key, value]) => `${key} = ${this.valueToSql(value)}`) .join(" AND "), ) .join(" OR ")}`; From 78480a78db745fe5834feb71913143cb015efe39 Mon Sep 17 00:00:00 2001 From: AgustinMJ Date: Mon, 11 Mar 2024 19:31:09 +0100 Subject: [PATCH 12/13] feat: index subscribed events only --- drills/ethers-typechain-contract/drill.ts | 118 +++++++++++++- .../src/EthersTypechainContractIndexer.ts | 9 +- packages/vanilla/src/Indexer.ts | 152 +++++++++++------- packages/vanilla/src/db/SQLiteDB.ts | 2 +- packages/vanilla/src/db/entities/LastEvent.ts | 24 +-- .../vanilla/src/db/entities/PendingEvent.ts | 30 ++-- .../vanilla/src/db/migrations/001-init.sql | 18 ++- .../account/src/XrplAccountIndexer.ts | 9 +- .../ledgers/src/XrplLedgersIndexer.ts | 7 +- 9 files changed, 274 insertions(+), 95 deletions(-) diff --git a/drills/ethers-typechain-contract/drill.ts b/drills/ethers-typechain-contract/drill.ts index e24a779..efb9fac 100644 --- a/drills/ethers-typechain-contract/drill.ts +++ b/drills/ethers-typechain-contract/drill.ts @@ -1,18 +1,126 @@ import { EthersTypechainContractIndexer } from "@bloxer/ethers-typechain-contract"; import { BridgeDoorCommon__factory } from "@peersyst/xrp-evm-contracts"; +import { appendFile } from "fs-extra"; async function drill() { const indexer = new EthersTypechainContractIndexer("0xa6262F3c85617a1Bd17d5a82153A2c6e69AEcfeD", BridgeDoorCommon__factory, { wsUrl: "ws://168.119.63.112:8546", persistenceFilePath: "persistence/.ethers-typechain-contract-indexer.db", - startingBlock: 4000000, + startingBlock: 4900000, + endingBlock: 6935496, + persist: true, + logger: { + minLevel: 3, + }, }); - indexer.on("CreateAccount", (event) => { - setTimeout(() => { - indexer.eventDone("CreateAccount", event.transactionHash, event.blockNumber); - }, 5000); + indexer.on("AddClaimAttestation", async (event) => { + console.log("Found AddClaimAttestation event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("AddClaimAttestation", event.transactionHash, event.logIndex); }); + indexer.on("AddCreateAccountAttestation", async (event) => { + console.log("Found AddCreateAccountAttestation event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("AddCreateAccountAttestation", event.transactionHash, event.logIndex); + }); + indexer.on("Claim", async (event) => { + console.log("Found Claim event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("Claim", event.transactionHash, event.logIndex); + }); + indexer.on("Commit", async (event) => { + console.log("Found Commit event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("Commit", event.transactionHash, event.logIndex); + }); + indexer.on("CommitWithoutAddress", async (event) => { + console.log("Found CommitWithoutAddress event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("CommitWithoutAddress", event.transactionHash, event.logIndex); + }); + indexer.on("CreateAccount", async (event) => { + console.log("Found CreateAccount event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("CreateAccount", event.transactionHash, event.logIndex); + }); + indexer.on("CreateAccountCommit", async (event) => { + console.log("Found CreateAccountCommit event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("CreateAccountCommit", event.transactionHash, event.logIndex); + }); + indexer.on("CreateBridge", async (event) => { + console.log("Found CreateBridge event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("CreateBridge", event.transactionHash, event.logIndex); + }); + indexer.on("CreateClaim", async (event) => { + console.log("Found CreateClaim event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("CreateClaim", event.transactionHash, event.logIndex); + }); + indexer.on("Credit", async (event) => { + console.log("Found Credit event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("Credit", event.transactionHash, event.logIndex); + }); + indexer.on("OwnershipTransferred", async (event) => { + console.log("Found OwnershipTransferred event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("OwnershipTransferred", event.transactionHash, event.logIndex); + }); + indexer.on("Paused", async (event) => { + console.log("Found Paused event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("Paused", event.transactionHash, event.logIndex); + }); + indexer.on("Unpaused", async (event) => { + console.log("Found Unpaused event"); + await appendFile( + "./dif-final-boss.txt", + `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, + ); + indexer.eventDone("Unpaused", event.transactionHash, event.logIndex); + }); + const startDate = new Date(); await indexer.run(); + const endDate = new Date(); + console.log(`Indexing took ${endDate.getTime() - startDate.getTime()} ms`); } drill(); diff --git a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts index 28c00ae..3b57760 100644 --- a/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts +++ b/packages/ethers/packages/typechain-contract/src/EthersTypechainContractIndexer.ts @@ -72,7 +72,14 @@ export class EthersTypechainContractIndexer { + /** + * The default config of the indexer. + */ private _defaultConfig: IndexerDefaultConfig = { startingBlock: 0, endingBlock: "latest", @@ -59,6 +62,9 @@ export abstract class Indexer { } } + /** + * The config of the indexer. + */ private _config: Required>; protected get config(): Required> { return this._config; @@ -67,10 +73,23 @@ export abstract class Indexer { this._config = value; } + /** + * The logger of the indexer. + */ readonly logger: Logger; + /** + * The event emitter of the indexer. + */ private readonly eventEmitter = new EventEmitter(); + /** + * Contains the active event listeners for each event. That is, the subscribed events using the `on` method. + */ + private readonly activeEventListeners: Partial> = {}; + /** + * Reference to the provider. + */ private _provider: Generics["provider"]; /** @@ -117,16 +136,6 @@ export abstract class Indexer { */ reachedLastEvent = false; - /** - * Event listener for the indexer - */ - readonly on: EventEmitter["on"]; - - /** - * Event emitter for the indexer - */ - protected readonly emit: EventEmitter["emit"]; - /** * Requests the provider with retries * Acts as a wrapper for provider.request @@ -145,8 +154,6 @@ export abstract class Indexer { this.logger = new Logger(this.config.logger); this.db = new SQLiteDB(this.config.persistenceFilePath); - this.on = this.eventEmitter.on.bind(this.eventEmitter); - this.emit = this.eventEmitter.emit.bind(this.eventEmitter); this.request = async function request(...args: any[]) { return this.withRetries( async () => { @@ -414,22 +421,20 @@ export abstract class Indexer { * Saves a new pending event by saving the last event and creating a new pending event. * Uses a mutex to ensure that only one pending event is saved at a time. * @param event The event to emit. - * @param hash The hash of the event. - * @param block The block of the event. - * @param data The data of the event. * @returns An array containing the last event and the pending event. */ - private async saveNewPendingEvent( - event: Event, - hash: string, - block: number, - ...data: Parameters - ): Promise<[LastEvent | void, PendingEvent]> { + private async saveNewPendingEvent({ + event, + hash, + i = 0, + block, + data, + }: PendingEvent>): Promise<[LastEvent | void, PendingEvent]> { return await this.eventMutex.runExclusive( async () => await Promise.all([ - this.db.getRepository(LastEvent).updateOrCreate(new LastEvent(block, event as string, hash)), - this.db.getRepository(PendingEvent).create(new PendingEvent(event as string, hash, block, ...data)), + this.db.getRepository(LastEvent).updateOrCreate({ block, event, hash, i }), + this.db.getRepository(PendingEvent).create({ event, hash, i, block, data }), ]), ); } @@ -442,42 +447,43 @@ export abstract class Indexer { */ private async saveBlock(block: number): Promise { return await this.eventMutex.runExclusive( - async () => await this.db.getRepository(LastEvent).updateOrCreate(new LastEvent(block + 1)), + async () => + await this.db.getRepository(LastEvent).updateOrCreate({ block: block + 1, event: undefined, hash: undefined, i: 0 }), ); } /** * Creates a pending event if `persist` is true and emits the event. * @param event The event to emit. - * @param hash The hash of the event. - * @param block The block of the event. - * @param data The data of the event. - */ - protected notifyEvent( - event: Event, - hash: string, - block: number, - ...data: Parameters - ): void { - if (this.config.persist) { - if (this.reachedLastEvent) { - this.saveNewPendingEvent(event, hash, block, ...data) - .then(() => { - this.emit(event, ...data); - }) - .catch((e) => { - this.logger.error( - `Error while saving new pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, - ); - }); - } else { - // We can assert `this.lastEvent` since it has to be defined for `this.reachedLastEvent` to be false. - if (this.lastEvent!.event === event && this.lastEvent!.hash === hash) { - this.reachedLastEvent = true; + */ + protected notifyEvent({ + event, + hash, + i, + block, + data, + }: PendingEvent>): void { + if (this.activeEventListeners[event]) { + if (this.config.persist) { + if (this.reachedLastEvent) { + this.saveNewPendingEvent({ event, hash, i, block, data }) + .then(() => { + this.emit(event, ...data); + }) + .catch((e) => { + this.logger.error( + `Error while saving new pending event ${event as string} with hash ${hash} and block ${block}: ${e}`, + ); + }); + } else { + // We can assert `this.lastEvent` since it has to be defined for `this.reachedLastEvent` to be false. + if (this.lastEvent!.event === event && this.lastEvent!.hash === hash && (!i || this.lastEvent.i! === i)) { + this.reachedLastEvent = true; + } } + } else { + this.emit(event, ...data); } - } else { - this.emit(event, ...data); } } @@ -533,10 +539,12 @@ export abstract class Indexer { * Recovers the pending events and emits them. */ private async recoverPendingEvents(): Promise { - const pendingEvents = await this.getPendingEvents(); + if (this.config.persist) { + const pendingEvents = await this.getPendingEvents(); - for (const pendingEvent of pendingEvents) { - this.emit(pendingEvent.event, ...(pendingEvent.data as Parameters)); + for (const pendingEvent of pendingEvents) { + this.emit(pendingEvent.event, ...(pendingEvent.data as Parameters)); + } } } @@ -547,14 +555,44 @@ export abstract class Indexer { * Does nothing when called with `persist` set to `false`. * @param event The event. * @param hash The hash of the event. - * @param block The block of the event. + * @param i The index of the event. Defaults to 0 since events without i are stored with i 0. */ - eventDone(event: Event, hash: string, block: number): void { + eventDone(event: Event, hash: string, i: number = 0): void { if (this.config.persist) { - this.db.getRepository(PendingEvent).delete({ event: event as string, hash, block }); + this.db.getRepository(PendingEvent).delete({ event, hash, i }); } } + /** + * Adds the `listener` function to the end of the listeners array for the specified event. + * @param event The event. + * @param listener The callback function. + * @returns A function that removes the listener. + */ + on(event: Event, listener: Generics["events"][Event]): () => void { + // Add the event to the active event listeners + this.activeEventListeners[event] = (this.activeEventListeners[event] ?? 0) + 1; + + const removeListener = this.eventEmitter.on(event, listener); + + return () => { + // Remove the event from the active event listeners + this.activeEventListeners[event] = (this.activeEventListeners[event] ?? 1) - 1; + + removeListener(); + }; + } + + /** + * Emits an event with the given arguments. + * @param event The event to emit. + * @param args The arguments to pass to the event listeners. + * @returns Returns `true` if the event had listeners, `false` otherwise. + */ + emit(event: Event, ...args: Parameters): boolean { + return this.eventEmitter.emit(event, ...args); + } + /** * Runs the indexer */ diff --git a/packages/vanilla/src/db/SQLiteDB.ts b/packages/vanilla/src/db/SQLiteDB.ts index b50dc38..2922d96 100644 --- a/packages/vanilla/src/db/SQLiteDB.ts +++ b/packages/vanilla/src/db/SQLiteDB.ts @@ -23,7 +23,7 @@ export class SQLiteDB implements DB { async open(): Promise { // Check if the database file exists, if not create it const existsDB = await exists(this.filename); - if (!existsDB) outputFile(this.filename, ""); + if (!existsDB) await outputFile(this.filename, ""); this.db = await openSQLite({ filename: this.filename, diff --git a/packages/vanilla/src/db/entities/LastEvent.ts b/packages/vanilla/src/db/entities/LastEvent.ts index 172dc79..9108429 100644 --- a/packages/vanilla/src/db/entities/LastEvent.ts +++ b/packages/vanilla/src/db/entities/LastEvent.ts @@ -6,15 +6,21 @@ import { Entity } from "./Entity"; * - Only `block`: Stores the upcoming block to be processed. * - `block`, `event`, `hash`: Stores the last event processed. The indexer will start indexing from the next event. */ -export class LastEvent extends Entity("last_event") { +export class LastEvent extends Entity("last_event") { + /** + * The block of the event. + */ block: number; - event?: string; + /** + * The event name. + */ + event?: Event; + /** + * The hash of the event. Normally the transaction hash. + */ hash?: string; - - constructor(block: number, event?: string, hash?: string) { - super(); - this.block = block; - this.event = event; - this.hash = hash; - } + /** + * The optional index of the event. + */ + i?: number; } diff --git a/packages/vanilla/src/db/entities/PendingEvent.ts b/packages/vanilla/src/db/entities/PendingEvent.ts index 60bb108..54657af 100644 --- a/packages/vanilla/src/db/entities/PendingEvent.ts +++ b/packages/vanilla/src/db/entities/PendingEvent.ts @@ -3,17 +3,25 @@ import { Entity } from "./Entity"; /** * Entity used to store the pending events to be processed. */ -export class PendingEvent extends Entity("pending_event") { - event: string; +export class PendingEvent extends Entity("pending_event") { + /** + * The event name. + */ + event: Event; + /** + * The hash of the event. Normally the transaction hash. + */ hash: string; + /** + * The optional index of the event. + */ + i?: number; + /** + * The block of the event. + */ block: number; - data: any[]; - - constructor(event: string, hash: string, block: number, ...data: any[]) { - super(); - this.event = event; - this.hash = hash; - this.block = block; - this.data = data; - } + /** + * The data of the event. + */ + data: Data; } diff --git a/packages/vanilla/src/db/migrations/001-init.sql b/packages/vanilla/src/db/migrations/001-init.sql index 424a57a..c967452 100644 --- a/packages/vanilla/src/db/migrations/001-init.sql +++ b/packages/vanilla/src/db/migrations/001-init.sql @@ -3,17 +3,19 @@ -------------------------------------------------------------------------------- CREATE TABLE pending_event ( - event TEXT, - hash TEXT, - block INTEGER NOT NULL, - data BLOB NOT NULL, - PRIMARY KEY (event, hash) + event TEXT NOT NULL, + hash TEXT NOT NULL, + i INTEGER DEFAULT 0 NOT NULL, + block INTEGER NOT NULL, + data BLOB, + PRIMARY KEY (event, hash, i) ); CREATE TABLE last_event ( - event TEXT, - hash TEXT, - block INTEGER NOT NULL + event TEXT, + hash TEXT, + i INTEGER DEFAULT 0, + block INTEGER NOT NULL ); -------------------------------------------------------------------------------- diff --git a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts index daff31b..e53cc1f 100644 --- a/packages/xrpl/packages/account/src/XrplAccountIndexer.ts +++ b/packages/xrpl/packages/account/src/XrplAccountIndexer.ts @@ -65,8 +65,13 @@ export class XrplAccountIndexer extends XrplIndexer<{ if (hash !== undefined && block !== undefined) { // Notify transaction events - this.notifyEvent("Transaction", hash, block, correctlyCastedAccountTx); - this.notifyEvent(tx.TransactionType, hash, block, correctlyCastedAccountTx as AccountTransaction); + this.notifyEvent({ event: "Transaction", hash, block, data: [correctlyCastedAccountTx] }); + this.notifyEvent({ + event: tx.TransactionType, + hash, + block, + data: [correctlyCastedAccountTx as AccountTransaction], + }); } } } diff --git a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts index b427a7d..52209e1 100644 --- a/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts +++ b/packages/xrpl/packages/ledgers/src/XrplLedgersIndexer.ts @@ -35,7 +35,12 @@ export class XrplLedgersIndexer extends XrplIndexer<{ }); if (res.result.validated) { - this.notifyEvent("Ledger", res.result.ledger_hash, res.result.ledger_index, res.result.ledger as SelectedLedgerType); + this.notifyEvent({ + event: "Ledger", + hash: res.result.ledger_hash, + block: res.result.ledger_index, + data: [res.result.ledger as SelectedLedgerType], + }); this.notifyBlock(res.result.ledger_index); ++currentBlock; } From f63158dfff1b0680b17fd8997524e884d17b0795 Mon Sep 17 00:00:00 2001 From: AdriaCarrera Date: Tue, 12 Mar 2024 09:40:18 +0100 Subject: [PATCH 13/13] fix: rename ethers contract event file --- drills/ethers-typechain-contract/drill.ts | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/drills/ethers-typechain-contract/drill.ts b/drills/ethers-typechain-contract/drill.ts index efb9fac..42fda46 100644 --- a/drills/ethers-typechain-contract/drill.ts +++ b/drills/ethers-typechain-contract/drill.ts @@ -16,7 +16,7 @@ async function drill() { indexer.on("AddClaimAttestation", async (event) => { console.log("Found AddClaimAttestation event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("AddClaimAttestation", event.transactionHash, event.logIndex); @@ -24,7 +24,7 @@ async function drill() { indexer.on("AddCreateAccountAttestation", async (event) => { console.log("Found AddCreateAccountAttestation event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("AddCreateAccountAttestation", event.transactionHash, event.logIndex); @@ -32,7 +32,7 @@ async function drill() { indexer.on("Claim", async (event) => { console.log("Found Claim event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("Claim", event.transactionHash, event.logIndex); @@ -40,7 +40,7 @@ async function drill() { indexer.on("Commit", async (event) => { console.log("Found Commit event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("Commit", event.transactionHash, event.logIndex); @@ -48,7 +48,7 @@ async function drill() { indexer.on("CommitWithoutAddress", async (event) => { console.log("Found CommitWithoutAddress event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("CommitWithoutAddress", event.transactionHash, event.logIndex); @@ -56,7 +56,7 @@ async function drill() { indexer.on("CreateAccount", async (event) => { console.log("Found CreateAccount event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("CreateAccount", event.transactionHash, event.logIndex); @@ -64,7 +64,7 @@ async function drill() { indexer.on("CreateAccountCommit", async (event) => { console.log("Found CreateAccountCommit event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("CreateAccountCommit", event.transactionHash, event.logIndex); @@ -72,7 +72,7 @@ async function drill() { indexer.on("CreateBridge", async (event) => { console.log("Found CreateBridge event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("CreateBridge", event.transactionHash, event.logIndex); @@ -80,7 +80,7 @@ async function drill() { indexer.on("CreateClaim", async (event) => { console.log("Found CreateClaim event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("CreateClaim", event.transactionHash, event.logIndex); @@ -88,7 +88,7 @@ async function drill() { indexer.on("Credit", async (event) => { console.log("Found Credit event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("Credit", event.transactionHash, event.logIndex); @@ -96,7 +96,7 @@ async function drill() { indexer.on("OwnershipTransferred", async (event) => { console.log("Found OwnershipTransferred event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("OwnershipTransferred", event.transactionHash, event.logIndex); @@ -104,7 +104,7 @@ async function drill() { indexer.on("Paused", async (event) => { console.log("Found Paused event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("Paused", event.transactionHash, event.logIndex); @@ -112,7 +112,7 @@ async function drill() { indexer.on("Unpaused", async (event) => { console.log("Found Unpaused event"); await appendFile( - "./dif-final-boss.txt", + "./bridge-events.txt", `${event.event} ${event.transactionHash} ${event.logIndex} ${event.blockNumber} ${JSON.stringify(event)}\n`, ); indexer.eventDone("Unpaused", event.transactionHash, event.logIndex);