feat!: optimize tables and improve canonical treatment of BNS data (#1287)

* refactor: re-order helper files

* fix: new columns for zonefiles

* feat: batch update subdomains

* fix: optimize some queries

* fix: api tests

* fix: optimize more queries, add cache handlers

* fix: some integration tests

* fix: all integration tests

* fix: v1 import test individually

* fix: move v1 import tests to separate file

* fix: start adding on conflict to subdomains

* fix: add uniqueness to names, subdomains and namespaces

* fix: remove extra index

* refactor: remove unused tx function

* fix: unused exports

* feat: echo block height while doing replays

* refactor: separate batch updates

* fix: calculate expire_block for a name based on namespace lifetime

* fix: test strict equal

* fix: remove secondary temporary table for event import

* fix: test sorting

* fix: detect namespaces readyed by third party contracts

* fix: bigint for namespaces

* refactor: move bns helper tests to bns suite

* chore: make event test less verbose

* chore: temporarily disable migrations on boot

* chore: add verbose logs to migration

* chore: make migration verbosity depend on log level

* fix: detect names from print event

* test: re-orgd attachments

* test: duplicate zonefile updates

Author: rafaelcr

src/api/routes/bns/addresses.ts

@@ -3,13 +3,20 @@ import { asyncHandler } from '../../async-handler';
 import { DataStore } from '../../../datastore/common';
 import { isUnanchoredRequest } from '../../query-helpers';
 import { ChainID } from '@stacks/transactions';
+import {
+  getETagCacheHandler,
+  setETagCacheHeaders,
+} from '../../../api/controllers/cache-controller';
 
 const SUPPORTED_BLOCKCHAINS = ['stacks'];
 
 export function createBnsAddressesRouter(db: DataStore, chainId: ChainID): express.Router {
   const router = express.Router();
+  const cacheHandler = getETagCacheHandler(db);
+
   router.get(
     '/:blockchain/:address',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       // Retrieves a list of names owned by the address provided.
       const { blockchain, address } = req.params;
@@ -23,6 +30,7 @@ export function createBnsAddressesRouter(db: DataStore, chainId: ChainID): expre
         includeUnanchored,
         chainId,
       });
+      setETagCacheHeaders(res);
       if (namesByAddress.found) {
         res.json({ names: namesByAddress.result });
       } else {

src/api/routes/bns/names.ts

@@ -3,92 +3,85 @@ import { asyncHandler } from '../../async-handler';
 import { DataStore } from '../../../datastore/common';
 import { parsePagingQueryInput } from '../../../api/pagination';
 import { isUnanchoredRequest } from '../../query-helpers';
-import { bnsBlockchain, BnsErrors } from '../../../bns-constants';
+import { bnsBlockchain, BnsErrors } from '../../../event-stream/bns/bns-constants';
 import { BnsGetNameInfoResponse } from '@stacks/stacks-blockchain-api-types';
 import { ChainID } from '@stacks/transactions';
+import {
+  getETagCacheHandler,
+  setETagCacheHeaders,
+} from '../../../api/controllers/cache-controller';
 
 export function createBnsNamesRouter(db: DataStore, chainId: ChainID): express.Router {
   const router = express.Router();
+  const cacheHandler = getETagCacheHandler(db);
 
   router.get(
     '/:name/zonefile/:zoneFileHash',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
-      // Fetches the historical zonefile specified by the username and zone hash.
       const { name, zoneFileHash } = req.params;
       const includeUnanchored = isUnanchoredRequest(req, res, next);
-      let nameFound = false;
-      const nameQuery = await db.getName({ name: name, includeUnanchored, chainId: chainId });
-      nameFound = nameQuery.found;
-      if (!nameFound) {
-        const subdomainQuery = await db.getSubdomain({ subdomain: name, includeUnanchored });
-        nameFound = subdomainQuery.found;
-      }
-
-      if (nameFound) {
-        const zonefile = await db.getHistoricalZoneFile({ name: name, zoneFileHash: zoneFileHash });
-        if (zonefile.found) {
-          res.json(zonefile.result);
-        } else {
-          res.status(404).json({ error: 'No such zonefile' });
-        }
+      const zonefile = await db.getHistoricalZoneFile({
+        name: name,
+        zoneFileHash: zoneFileHash,
+        includeUnanchored,
+      });
+      if (zonefile.found) {
+        setETagCacheHeaders(res);
+        res.json(zonefile.result);
       } else {
-        res.status(400).json({ error: 'Invalid name or subdomain' });
+        res.status(404).json({ error: 'No such name or zonefile' });
       }
     })
   );
 
   router.get(
     '/:name/subdomains',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       const { name } = req.params;
       const includeUnanchored = isUnanchoredRequest(req, res, next);
       const subdomainsList = await db.getSubdomainsListInName({ name, includeUnanchored });
+      setETagCacheHeaders(res);
       res.json(subdomainsList.results);
     })
   );
 
   router.get(
     '/:name/zonefile',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
-      // Fetch a user’s raw zone file. This only works for RFC-compliant zone files. This method returns an error for names that have non-standard zone files.
       const { name } = req.params;
       const includeUnanchored = isUnanchoredRequest(req, res, next);
-      let nameFound = false;
-      const nameQuery = await db.getName({ name: name, includeUnanchored, chainId: chainId });
-      nameFound = nameQuery.found;
-      if (!nameFound) {
-        const subdomainQuery = await db.getSubdomain({ subdomain: name, includeUnanchored });
-        nameFound = subdomainQuery.found;
-      }
-
-      if (nameFound) {
-        const zonefile = await db.getLatestZoneFile({ name: name, includeUnanchored });
-        if (zonefile.found) {
-          res.json(zonefile.result);
-        } else {
-          res.status(404).json({ error: 'No zone file for name' });
-        }
+      const zonefile = await db.getLatestZoneFile({ name: name, includeUnanchored });
+      if (zonefile.found) {
+        setETagCacheHeaders(res);
+        res.json(zonefile.result);
       } else {
-        res.status(400).json({ error: 'Invalid name or subdomain' });
+        res.status(404).json({ error: 'No such name or zonefile does not exist' });
       }
     })
   );
 
   router.get(
     '/',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       const page = parsePagingQueryInput(req.query.page ?? 0);
       const includeUnanchored = isUnanchoredRequest(req, res, next);
       const { results } = await db.getNamesList({ page, includeUnanchored });
       if (results.length === 0 && req.query.page) {
         res.status(400).json(BnsErrors.InvalidPageNumber);
+      } else {
+        setETagCacheHeaders(res);
+        res.json(results);
       }
-      res.json(results);
     })
   );
 
   router.get(
     '/:name',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       const { name } = req.params;
       const includeUnanchored = isUnanchoredRequest(req, res, next);
@@ -149,6 +142,7 @@ export function createBnsNamesRouter(db: DataStore, chainId: ChainID): express.R
       const response = Object.fromEntries(
         Object.entries(nameInfoResponse).filter(([_, v]) => v != null)
       );
+      setETagCacheHeaders(res);
       res.json(response);
     })
   );

src/api/routes/bns/namespaces.ts

@@ -3,27 +3,35 @@ import { asyncHandler } from '../../async-handler';
 import { DataStore } from '../../../datastore/common';
 import { parsePagingQueryInput } from '../../../api/pagination';
 import { isUnanchoredRequest } from '../../query-helpers';
-import { BnsErrors } from '../../../bns-constants';
+import { BnsErrors } from '../../../event-stream/bns/bns-constants';
 import { BnsGetAllNamespacesResponse } from '@stacks/stacks-blockchain-api-types';
+import {
+  getETagCacheHandler,
+  setETagCacheHeaders,
+} from '../../../api/controllers/cache-controller';
 
 export function createBnsNamespacesRouter(db: DataStore): express.Router {
   const router = express.Router();
+  const cacheHandler = getETagCacheHandler(db);
 
   router.get(
     '/',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       const includeUnanchored = isUnanchoredRequest(req, res, next);
       const { results } = await db.getNamespaceList({ includeUnanchored });
       const response: BnsGetAllNamespacesResponse = {
         namespaces: results,
       };
+      setETagCacheHeaders(res);
       res.json(response);
       return;
     })
   );
 
   router.get(
     '/:tld/names',
+    cacheHandler,
     asyncHandler(async (req, res, next) => {
       const { tld } = req.params;
       const page = parsePagingQueryInput(req.query.page ?? 0);
@@ -39,8 +47,10 @@ export function createBnsNamespacesRouter(db: DataStore): express.Router {
         });
         if (results.length === 0 && req.query.page) {
           res.status(400).json(BnsErrors.InvalidPageNumber);
+        } else {
+          setETagCacheHeaders(res);
+          res.json(results);
         }
-        res.json(results);
       }
     })
   );

src/api/routes/bns/pricing.ts

@@ -14,12 +14,12 @@ import {
   listCV,
   ChainID,
 } from '@stacks/transactions';
-import { GetStacksNetwork, getBnsContractID } from './../../../bns-helpers';
 import {
   BnsGetNamePriceResponse,
   BnsGetNamespacePriceResponse,
 } from '@stacks/stacks-blockchain-api-types';
 import { isValidPrincipal, logger } from './../../../helpers';
+import { getBnsContractID, GetStacksNetwork } from '../../../event-stream/bns/bns-helpers';
 
 export function createBnsPriceRouter(db: DataStore, chainId: ChainID): express.Router {
   const router = express.Router();

src/datastore/common.ts

@@ -428,6 +428,31 @@ export interface DataStoreTxEventData {
   namespaces: DbBnsNamespace[];
 }
 
+export interface DataStoreAttachmentData {
+  op: string;
+  name: string;
+  namespace: string;
+  zonefile: string;
+  zonefileHash: string;
+  txId: string;
+  indexBlockHash: string;
+  blockHeight: number;
+}
+
+export interface DataStoreSubdomainBlockData {
+  index_block_hash: string;
+  parent_index_block_hash: string;
+  microblock_hash: string;
+  microblock_sequence: number;
+  microblock_canonical: boolean;
+}
+
+export interface DataStoreAttachmentSubdomainData {
+  attachment?: DataStoreAttachmentData;
+  blockData?: DataStoreSubdomainBlockData;
+  subdomains?: DbBnsSubdomain[];
+}
+
 export interface DbSearchResult {
   entity_type: 'standard_address' | 'contract_address' | 'block_hash' | 'tx_id' | 'mempool_tx_id';
   entity_id: string;
@@ -472,6 +497,7 @@ export interface DbInboundStxTransfer {
 export interface DbBnsZoneFile {
   zonefile: string;
 }
+
 export interface DbBnsNamespace {
   id?: number;
   namespace_id: string;
@@ -480,8 +506,8 @@ export interface DbBnsNamespace {
   reveal_block: number;
   ready_block: number;
   buckets: string;
-  base: number;
-  coeff: number;
+  base: bigint;
+  coeff: bigint;
   nonalpha_discount: number;
   no_vowel_discount: number;
   lifetime: number;
@@ -683,7 +709,6 @@ export interface DataStore extends DataStoreEventEmitter {
     limit: number;
     offset: number;
   }): Promise<{ results: DbMempoolTx[]; total: number }>;
-  getTxStrict(args: { txId: string; indexBlockHash: string }): Promise<FoundOrNot<DbTx>>;
   getTx(args: { txId: string; includeUnanchored: boolean }): Promise<FoundOrNot<DbTx>>;
   getTxList(args: {
     limit: number;
@@ -744,7 +769,8 @@ export interface DataStore extends DataStoreEventEmitter {
 
   updateMicroblocks(data: DataStoreMicroblockUpdateData): Promise<void>;
 
-  updateZoneContent(zonefile: string, zonefile_hash: string, tx_id: string): Promise<void>;
+  updateAttachments(attachments: DataStoreAttachmentData[]): Promise<void>;
+
   resolveBnsSubdomains(
     blockData: {
       index_block_hash: string;
@@ -948,6 +974,7 @@ export interface DataStore extends DataStoreEventEmitter {
   getHistoricalZoneFile(args: {
     name: string;
     zoneFileHash: string;
+    includeUnanchored: boolean;
   }): Promise<FoundOrNot<DbBnsZoneFile>>;
   getLatestZoneFile(args: {
     name: string;