From a351ca410d754bceeae383cb2f1344c7b6af6379 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 29 Aug 2023 23:15:43 -0600 Subject: [PATCH 1/3] fix: use cursor query for brc-20 scan --- src/pg/brc20/brc20-pg-store.ts | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/src/pg/brc20/brc20-pg-store.ts b/src/pg/brc20/brc20-pg-store.ts index a2423497..e6fb82f2 100644 --- a/src/pg/brc20/brc20-pg-store.ts +++ b/src/pg/brc20/brc20-pg-store.ts @@ -49,16 +49,10 @@ export class Brc20PgStore { * @param startBlock - Start at block height * @param endBlock - End at block height */ - async scanBlocks(startBlock?: number, endBlock?: number): Promise { - const range = await this.parent.sql<{ min: number; max: number }[]>` - SELECT - ${startBlock ? this.parent.sql`${startBlock}` : this.parent.sql`MIN(block_height)`} AS min, - ${endBlock ? this.parent.sql`${endBlock}` : this.parent.sql`MAX(block_height)`} AS max - FROM locations - `; - for (let blockHeight = range[0].min; blockHeight <= range[0].max; blockHeight++) { + async scanBlocks(startBlock: number, endBlock: number): Promise { + for (let blockHeight = startBlock; blockHeight <= endBlock; blockHeight++) { await this.parent.sqlWriteTransaction(async sql => { - const block = await sql` + const cursor = sql` SELECT i.content, EXISTS(SELECT location_id FROM genesis_locations WHERE location_id = l.id) AS genesis, @@ -69,8 +63,10 @@ export class Brc20PgStore { AND i.number >= 0 AND i.mime_type IN ('application/json', 'text/plain') ORDER BY tx_index ASC - `; - await this.insertOperations(block); + `.cursor(); + for await (const chunk of cursor) { + await this.insertOperations(chunk); + } }); } } From 334e3611f7b278d7d41db5de9d83fa35b149de61 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Tue, 29 Aug 2023 23:55:34 -0600 Subject: [PATCH 2/3] fix: optimize content retrieval --- src/pg/brc20/brc20-pg-store.ts | 14 +++++++------- src/pg/brc20/types.ts | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/pg/brc20/brc20-pg-store.ts b/src/pg/brc20/brc20-pg-store.ts index e6fb82f2..0c950f75 100644 --- a/src/pg/brc20/brc20-pg-store.ts +++ b/src/pg/brc20/brc20-pg-store.ts @@ -52,9 +52,8 @@ export class Brc20PgStore { async scanBlocks(startBlock: number, endBlock: number): Promise { for (let blockHeight = startBlock; blockHeight <= endBlock; blockHeight++) { await this.parent.sqlWriteTransaction(async sql => { - const cursor = sql` + const block = await sql` SELECT - i.content, EXISTS(SELECT location_id FROM genesis_locations WHERE location_id = l.id) AS genesis, ${sql(LOCATIONS_COLUMNS.map(c => `l.${c}`))} FROM locations AS l @@ -63,10 +62,8 @@ export class Brc20PgStore { AND i.number >= 0 AND i.mime_type IN ('application/json', 'text/plain') ORDER BY tx_index ASC - `.cursor(); - for await (const chunk of cursor) { - await this.insertOperations(chunk); - } + `; + await this.insertOperations(block); }); } } @@ -76,7 +73,10 @@ export class Brc20PgStore { for (const write of writes) { if (write.genesis) { if (write.address === null) continue; - const brc20 = brc20FromInscriptionContent(hexToBuffer(write.content)); + const content = await this.parent.sql<{ content: string }[]>` + SELECT content FROM inscriptions WHERE id = ${write.inscription_id} + `; + const brc20 = brc20FromInscriptionContent(hexToBuffer(content[0].content)); if (brc20) { switch (brc20.op) { case 'deploy': diff --git a/src/pg/brc20/types.ts b/src/pg/brc20/types.ts index 9ff859a3..741b7adf 100644 --- a/src/pg/brc20/types.ts +++ b/src/pg/brc20/types.ts @@ -3,7 +3,6 @@ import { DbLocation } from '../types'; export type DbBrc20ScannedInscription = DbLocation & { genesis: boolean; - content: string; }; export type DbBrc20DeployInsert = { From 3766f9d7451c4a29576dd9f38bd49b2a69e64040 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 30 Aug 2023 00:10:25 -0600 Subject: [PATCH 3/3] fix: ignore transfers that are not brc-20 --- src/pg/brc20/brc20-pg-store.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pg/brc20/brc20-pg-store.ts b/src/pg/brc20/brc20-pg-store.ts index 0c950f75..b7ec1043 100644 --- a/src/pg/brc20/brc20-pg-store.ts +++ b/src/pg/brc20/brc20-pg-store.ts @@ -73,6 +73,7 @@ export class Brc20PgStore { for (const write of writes) { if (write.genesis) { if (write.address === null) continue; + // Read contents here to avoid OOM errors when bulk-requesting content from postgres. const content = await this.parent.sql<{ content: string }[]>` SELECT content FROM inscriptions WHERE id = ${write.inscription_id} `; @@ -231,7 +232,7 @@ export class Brc20PgStore { OR (l.block_height = ${args.block_height} AND l.tx_index < ${args.tx_index})) LIMIT 3 `; - if (brc20Transfer.count > 2) return; + if (brc20Transfer.count === 0 || brc20Transfer.count > 2) return; const transfer = brc20Transfer[0]; const amount = new BigNumber(transfer.amount); const changes: DbBrc20BalanceInsert[] = [