feat(pLimit): add p-limit to limit concurrency of requests and better…

… erorr handling for db inserts

package.json

@@ -10,7 +10,6 @@
   },
   "dependencies": {
     "@ar.io/sdk": "^1.0.0-alpha.12",
-    "@ardrive/ardrive-promise-cache": "^1.1.4",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
@@ -19,6 +18,7 @@
     "express-prometheus-middleware": "^1.2.0",
     "lmdb": "^3.0.0",
     "middleware-async": "^1.3.5",
+    "p-limit": "4",
     "prom-client": "^14.0.1",
     "swagger-ui-express": "^5.0.0",
     "winston": "^3.7.2",

src/server.ts

@@ -96,7 +96,6 @@ app.get('/ar-io/resolver/records/:name', async (req, res) => {
       return;
     }
     const recordData = JSON.parse(resolvedRecordData.toString());
-    console.log('recordData', recordData)
     res
       .status(200)
       .set({

src/system.ts

@@ -16,6 +16,7 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 import { ANT, ANTRecord, ArIO, RemoteContract } from '@ar.io/sdk/node';
+import pLimit from 'p-limit';
 
 import { LmdbKVStore } from './cache/lmdb-kv-store.js';
 import * as config from './config.js';
@@ -37,6 +38,8 @@ export const cache = new LmdbKVStore({
   ttlSeconds: config.EVALUATION_INTERVAL_MS / 1000,
 });
 
+const parallelLimit = pLimit(100);
+
 export async function evaluateArNSNames() {
   log.info('Evaluating arns names');
 
@@ -58,28 +61,30 @@ export async function evaluateArNSNames() {
   > = {};
   // TODO: wrap this in p-limit to avoid overloading the node process
   await Promise.all(
-    [...contractTxIds].map(async (contractTxId) => {
-      const antContract = new ANT({ contractTxId });
-      const antRecords = await antContract.getRecords().catch((err) => {
-        log.error('Failed to get records for contract', {
-          contractTxId,
-          error: err,
+    [...contractTxIds].map((contractTxId) => {
+      return parallelLimit(async () => {
+        const antContract = new ANT({ contractTxId });
+        const antRecords = await antContract.getRecords().catch((err) => {
+          log.error('Failed to get records for contract', {
+            contractTxId,
+            error: err,
+          });
+          return {};
         });
-        return {};
-      });
 
-      if (Object.keys(antRecords).length) {
-        contractRecordMap[contractTxId] = {
-          owner: await antContract.getOwner().catch((err) => {
-            log.error('Failed to get owner for contract', {
-              contractTxId,
-              error: err,
-            });
-            return undefined;
-          }),
-          records: antRecords,
-        };
-      }
+        if (Object.keys(antRecords).length) {
+          contractRecordMap[contractTxId] = {
+            owner: await antContract.getOwner().catch((err) => {
+              log.error('Failed to get owner for contract', {
+                contractTxId,
+                error: err,
+              });
+              return undefined;
+            }),
+            records: antRecords,
+          };
+        }
+      });
     }),
   );
 
@@ -116,11 +121,19 @@ export async function evaluateArNSNames() {
         JSON.stringify(resolvedRecordObj),
       );
       const cacheKey = antName === '@' ? apexName : `${antName}_${apexName}`;
-      insertPromises.push(cache.set(cacheKey, resolvedRecordBuffer));
+      const promise = cache.set(cacheKey, resolvedRecordBuffer).catch((err) => {
+        log.error('Failed to set record in cache', {
+          cacheKey,
+          error: err,
+        });
+      });
+      insertPromises.push(promise);
     }
   }
-  // await all the inserts - performance concerns on the db
-  await Promise.all(insertPromises);
+  // await all the inserts - use plimit to hammering the cache
+  await Promise.all(
+    insertPromises.map((promise) => parallelLimit(() => promise)),
+  );
 
   // TODO: clean up any records that are no longer valid
   log.info('Successfully evaluated arns names', {

yarn.lock

@@ -7,13 +7,6 @@
   resolved "https://registry.yarnpkg.com/@aashutoshrathi/word-wrap/-/word-wrap-1.2.6.tgz#bd9154aec9983f77b3a034ecaa015c2e4201f6cf"
   integrity sha512-1Yjs2SvM8TflER/OD3cOjhWWOZb58A2t7wpE2S9XfBYTiIl+XFhQG2bjy4Pu1I+EAlCNUzRDYDdFwFYUKvXcIA==
 
-"@alexsasharegan/simple-cache@^3.3.3":
-  version "3.3.3"
-  resolved "https://registry.yarnpkg.com/@alexsasharegan/simple-cache/-/simple-cache-3.3.3.tgz#6e17ea1caee06e62857c704bc14630735cf0dc16"
-  integrity sha512-SWLaNj75VckhB7tFmfD4vqORVKmwqCYiZTfdTLU87aMF9mkpHp1UeSNKM2axZZDTIDqx45msonjBYeMvhE+amQ==
-  dependencies:
-    safe-types "^4.17.0"
-
 "@ampproject/remapping@^2.1.0":
   version "2.2.1"
   resolved "https://registry.yarnpkg.com/@ampproject/remapping/-/remapping-2.2.1.tgz#99e8e11851128b8702cd57c33684f1d0f260b630"
@@ -51,13 +44,6 @@
     warp-contracts "^1.4.38"
     winston "^3.11.0"
 
-"@ardrive/ardrive-promise-cache@^1.1.4":
-  version "1.1.4"
-  resolved "https://registry.yarnpkg.com/@ardrive/ardrive-promise-cache/-/ardrive-promise-cache-1.1.4.tgz#6199aaf4fc861e74a7d355c81321883a9abb3b8b"
-  integrity sha512-8HzOfgY7CVFJNJX/xZSbbZIl7qfO+Mkbp100bNdB/e40YrFFXfy1NXxQR2lc/AJKsMGdcWgSqfI8QOeFYQup1A==
-  dependencies:
-    "@alexsasharegan/simple-cache" "^3.3.3"
-
 "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.5":
   version "7.22.10"
   resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.10.tgz#1c20e612b768fefa75f6e90d6ecb86329247f0a3"
@@ -4416,6 +4402,13 @@ osenv@^0.1.4:
     os-homedir "^1.0.0"
     os-tmpdir "^1.0.0"
 
+p-limit@4:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-4.0.0.tgz#914af6544ed32bfa54670b061cafcbd04984b644"
+  integrity sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==
+  dependencies:
+    yocto-queue "^1.0.0"
+
 p-limit@^2.2.0:
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-2.3.0.tgz#3dd33c647a214fdfffd835933eb086da0dc21db1"
@@ -4857,11 +4850,6 @@ safe-stable-stringify@^2.3.1:
   resolved "https://registry.yarnpkg.com/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz#138c84b6f6edb3db5f8ef3ef7115b8f55ccbf886"
   integrity sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==
 
-safe-types@^4.17.0:
-  version "4.27.0"
-  resolved "https://registry.yarnpkg.com/safe-types/-/safe-types-4.27.0.tgz#6016efdf32ddc29b052f77d15ce3fa4c734b8841"
-  integrity sha512-6FY9SBSfWIaLKIi+wHKwdKHQy1rJMUdlsZVleXbMX35bSC2j5Jul4o59Sl7S+argcFCQXLurS0DC+CDFKA9cIw==
-
 "safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
@@ -5684,6 +5672,11 @@ yocto-queue@^0.1.0:
   resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"
   integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==
 
+yocto-queue@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.0.0.tgz#7f816433fb2cbc511ec8bf7d263c3b58a1a3c251"
+  integrity sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==
+
 zip-stream@^4.1.0:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/zip-stream/-/zip-stream-4.1.1.tgz#1337fe974dbaffd2fa9a1ba09662a66932bd7135"