Merge pull request #314 from kodadot/main

🔖 Stick v12

README.md

@@ -162,6 +162,14 @@ CHAIN=polkadot # or kusama
 OFFER=<ID_OF_THE_COLLECTION>
 ```
 
+6. debugging the processor
+
+As the processor can run for a longer period of time it is useful to turn off the "features" that you do not need,
+Handlers that need to be always enabled are `createCollection` and `createItem` as they are the base for the rest of the processor.
+
+> [!NOTE]
+> If you do not wish to index `uniques` pallet you can turn it off by setting `UNIQUES_ENABLED=false` in `.env` file
+
 ### Note on Swaps
 
 1. Swaps can be overwritten at any time

db/migrations/1726485003107-Data.js

@@ -0,0 +1,13 @@
+module.exports = class Data1726485003107 {
+ name = 'Data1726485003107'
+
+ async up(db) {
+ await db.query(`ALTER TABLE "metadata_entity" ADD "kind" character varying(6)`)
+ await db.query(`ALTER TABLE "collection_entity" ADD "kind" character varying(6)`)
+ }
+
+ async down(db) {
+ await db.query(`ALTER TABLE "metadata_entity" DROP COLUMN "kind"`)
+ await db.query(`ALTER TABLE "collection_entity" DROP COLUMN "kind"`)
+ }
+}

package.json

@@ -16,13 +16,13 @@
  "query-node:start": "squid-graphql-server --subscriptions --max-response-size 10000 --dumb-cache in-memory --dumb-cache-ttl 1000 --dumb-cache-size 100 --dumb-cache-max-age 1000"
  },
  "dependencies": {
- "@kodadot1/hyperdata": "^0.0.1-rc.4",
+ "@kodadot1/hyperdata": "^0.0.1-rc.5",
  "@kodadot1/metasquid": "^0.3.1-rc.0",
  "@kodadot1/minipfs": "^0.4.3-rc.2",
  "@subsquid/archive-registry": "3.3.2",
- "@subsquid/graphql-server": "4.6.0",
+ "@subsquid/graphql-server": "4.7.0",
  "@subsquid/ss58": "2.0.2",
- "@subsquid/substrate-processor": "8.4.1",
+ "@subsquid/substrate-processor": "8.5.1",
  "@subsquid/typeorm-migration": "1.3.0",
  "@subsquid/typeorm-store": "1.5.1",
  "dotenv": "^16.4.5",
@@ -33,9 +33,9 @@
  "typeorm": "0.3.17"
  },
  "devDependencies": {
- "@subsquid/substrate-metadata-explorer": "3.1.2",
+ "@subsquid/substrate-metadata-explorer": "3.2.0",
  "@subsquid/substrate-typegen": "8.1.0",
- "@subsquid/typeorm-codegen": "2.0.1",
+ "@subsquid/typeorm-codegen": "2.0.2",
  "@types/md5": "^2.3.5",
  "@types/node": "18.11.18",
  "@types/pg": "^8.11.4",

schema.graphql

@@ -16,6 +16,7 @@ type CollectionEntity @entity {
  id: ID!
  image: String
  issuer: String!
+ kind: Kind
  max: Int
  media: String
  meta: MetadataEntity
@@ -97,6 +98,7 @@ type MetadataEntity @entity {
  animationUrl: String
  type: String
  banner: String
+ kind: Kind
 }
 
 # Entity to represent an Attribute
@@ -259,6 +261,15 @@ enum TradeStatus {
  WITHDRAWN
 }
 
+enum Kind {
+ poap
+ pfp
+ genart
+ mixed
+ # audio
+ # video
+}
+
 # Entity to represent a Fungible Asset
 # defined on chain as pub type Asset<T: Config<I>, I: 'static = ()>
 # https://github.com/paritytech/polkadot-sdk/blob/99234440f0f8b24f7e4d1d3a0102a9b19a408dd3/substrate/frame/assets/src/lib.rs#L325

speck.yaml

@@ -1,6 +1,6 @@
 manifestVersion: subsquid.io/v0.1
 name: speck
-version: 13
+version: 14
 description: 'SubSquid indexer for Uniques and Assets on Statemint'
 build: 
 deploy:
@@ -16,6 +16,7 @@ deploy:
  env:
  CHAIN: polkadot
  OFFER: 174
+ UNIQUES_ENABLED: true
  api:
  cmd:
  - npx

squid.yaml

@@ -16,6 +16,7 @@ deploy:
  env:
  CHAIN: kusama
  OFFER: 464
+ UNIQUES_ENABLED: true
  api:
  cmd:
  - npx

src/environment.ts

@@ -3,9 +3,10 @@ export type Chain = 'kusama' | 'rococo' | 'polkadot'
 export const CHAIN: Chain = process.env.CHAIN as Chain || 'kusama'
 export const COLLECTION_OFFER: string = process.env.OFFER || ''
 
+export const UNIQUES_ENABLED = process.env.UNIQUES === 'true'
 const UNIQUE_STARTING_BLOCK = 323_750 // 618838;
-// const _NFT_STARTING_BLOCK = 4_556_552
-export const STARTING_BLOCK = UNIQUE_STARTING_BLOCK
+const NFT_STARTING_BLOCK = 4_556_552
+export const STARTING_BLOCK = UNIQUES_ENABLED ? UNIQUE_STARTING_BLOCK : NFT_STARTING_BLOCK
 
 // Asset Hub
 const ARCHIVE_URL = `https://v2.archive.subsquid.io/network/asset-hub-${CHAIN}`
@@ -16,6 +17,7 @@ export const isProd = CHAIN !== 'rococo'
 console.table({
  CHAIN, ARCHIVE_URL, NODE_URL, STARTING_BLOCK,
  COLLECTION_OFFER,
+ UNIQUES_ENABLED,
  disabledRPC: false,
  environment: isProd ? 'production' : 'development',
 })

src/mappings/index.ts

@@ -8,8 +8,7 @@ import * as a from './assets'
 import * as n from './nfts'
 import * as u from './uniques'
 import { BatchContext, Context, SelectedEvent } from './utils/types'
-import { updateCache } from './utils/cache'
-import { logError } from './utils/logger'
+import { updateOfferCache } from './utils/cache'
 
 type HandlerFunction = <T extends SelectedEvent>(item: T, ctx: Context) => Promise<void>
 
@@ -234,105 +233,26 @@ export async function mainFrame(ctx: BatchContext<Store>): Promise<void> {
  }
  }
 
- // const lastDate = new Date(ctx.blocks[ctx.blocks.length - 1].header.timestamp || start)
- // await updateCache(lastDate, ctx.store) 
-
- // const { contracts, tokens } = uniqueEntitySets(items)
- // const collections = await finalizeCollections(contracts, ctx)
- // const finish = await whatToDoWithTokens({ tokens, collections, items }, ctx)
- // const complete = await completeTokens(ctx, finish)
-
- // logger.info(`Batch completed, ${complete.length} tokens saved`)
+ if (ctx.isHead) {
+ const lastBlock = ctx.blocks[ctx.blocks.length - 1].header
+ const lastDate = new Date(lastBlock.timestamp || Date.now())
+ logger.info(`Found head block, updating cache`)
+ await updateOfferCache(lastDate, lastBlock.height, ctx.store) 
+ }
 }
 
-// function unwrapLog(log: Log, block: BlockData) {
-// switch (log.topics[0]) {
-// case ERC721_TRANSFER:
+// class Head {
+// #height: number
 
-// if (log.address !== Contracts.HueNft) {
-// return null
-// }
-// return handle721Token(log, block)
-// default:
-// // console.log('unknown log', log.topics[0])
-// return null
-// // throw new Error('unknown log')
+// constructor(height: number) {
+// this.#height = height
 // }
-// }
-
-type What = {
- // tokens: Set<string>,
- // collections: EnMap<CE>,
- // items: ItemStateUpdate[],
-}
 
-export async function whatToDoWithTokens(x: What, ctx: Context) {
- // // ctx.store.findBy(CE, {id: In([...collectionMap.keys()])})
- // const knownTokens = await findByIdListAsMap(ctx.store, NE, tokens)
- // const events: EventEntity[] = []
- // for (const item of items) {
- // logger.debug(`APPLY ${item.interaction} on ${item.id}`)
- // let knownToken = knownTokens.get(item.id) ?? create(NE, item.id, {})
- // if (item.applyFrom) {
- // const collection = collections.get(item.contract)!
- // item.applyFrom(collection)
- // }
- // if (item.applyTo) {
- // knownToken = item.applyTo(knownToken)
- // }
- // events.push(item.event)
- // knownTokens.set(item.id, knownToken)
- // }
- // const values = [...knownTokens.values()]
- // await ctx.store.upsert(values)
- // await ctx.store.save(events)
- // return knownTokens
-}
+// get height() {
+// return this.#height
+// }
 
-type EnMap<T> = Map<string, T>
-// TODO: do only if event was mint.
-async function completeTokens(ctx: Context, tokenMap: EnMap<NE>) {
- // const collections = groupedItemsByCollection(tokenMap.keys())
- // const final: NE[] = []
- // const metadataFutures: Promise<Optional<MetadataEntity>>[] = []
- // for (const [contract, ids] of collections.entries()) {
- // const list = Array.from(ids)
- // const tokens = await multicallMetadataFetch(ctx, contract, list)
- // for (const [i, id] of list.entries()) {
- // const realId = createTokenId(contract, id)
- // const token = tokenMap.get(realId)!
- // if (!token.metadata) {
- // const metadata = tokens[i]
- // token.metadata = metadata
- // const getMeta = handleMetadata(metadata, ctx.store).then(m => {
- // if (m) {
- // token.meta = m
- // token.name = m.name
- // token.image = m.image
- // token.media = m.animationUrl
- // }
- // return m
- // })
- // metadataFutures.push(getMeta)
- // final.push(token)
- // }
- // }
- // }
- // const metaList = await Promise.all(metadataFutures)
- // const filtered = metaList.filter(m => m) as MetadataEntity[]
- // logger.debug(`Saving ${filtered.length} metadata`)
- // await ctx.store.save(filtered)
- // await ctx.store.save(final)
- // return final
- // }
- // async function multicallMetadataFetch(ctx: Context, collection: string, tokens: Array<string>): Promise<string[]> {
- // const tokenIds = tokens.map((id) => [BigInt(id)])
- // const contract = new Multicall(ctx, lastBatchBlock(ctx), MULTICALL_ADDRESS)
- // const metadata = await contract.aggregate(
- // erc721.functions.tokenURI,
- // collection,
- // tokenIds,
- // MULTICALL_BATCH_SIZE
- // )
- // return metadata
-}
+// set height(height: number) {
+// this.#height = height
+// }
+// }

src/mappings/nfts/buy.ts

@@ -26,7 +26,7 @@ export async function handleTokenBuy(context: Context): Promise<void> {
  const originalPrice = event.price
  const originalOwner = entity.currentOwner ?? undefined
 
- entity.price = BigInt(0)
+ entity.price = null
  entity.currentOwner = event.caller
  entity.updatedAt = event.timestamp
 

src/mappings/nfts/create.ts

@@ -1,6 +1,6 @@
 import { getOrCreate } from '@kodadot1/metasquid/entity'
 import md5 from 'md5'
-import { CollectionEntity as CE } from '../../model'
+import { CollectionEntity as CE, Kind } from '../../model'
 import { handleMetadata } from '../shared/metadata'
 import { unwrap } from '../utils/extract'
 import { debug, pending, success } from '../utils/logger'
@@ -50,6 +50,7 @@ export async function handleCollectionCreate(context: Context): Promise<void> {
  final.name = metadata?.name
  final.image = metadata?.image
  final.media = metadata?.animationUrl
+ final.kind = metadata?.kind || Kind.mixed
  }
 
  success(OPERATION, `[COLLECTION] ${final.id}`)

src/mappings/nfts/list.ts

@@ -23,7 +23,7 @@ export async function handleTokenList(context: Context): Promise<void> {
  const id = createTokenId(event.collectionId, event.sn)
  const entity = await getWith(context.store, NE, id, { collection: true })
 
- entity.price = event.price
+ entity.price = Boolean(event.price) ? event.price : null
 
  if (event.price && (entity.collection.floor === 0n || event.price < entity.collection.floor)) {
  entity.collection.floor = event.price

src/mappings/nfts/mint.ts

@@ -43,7 +43,7 @@ export async function handleTokenCreate(context: Context): Promise<void> {
  final.collection = collection
  final.sn = BigInt(event.sn)
  final.metadata = event.metadata || tokenUri(collection.baseUri, event.sn) || collection.metadata
- final.price = BigInt(0)
+ final.price = null
  final.burned = false
  final.createdAt = event.timestamp
  final.updatedAt = event.timestamp

src/mappings/nfts/transfer.ts

@@ -41,7 +41,7 @@ export async function handleTokenTransfer(context: Context): Promise<void> {
  const entity = await getWith(context.store, NE, id, { collection: true })
 
  const oldOwner = entity.currentOwner
- entity.price = BigInt(0)
+ entity.price = null
  entity.currentOwner = event.to
  entity.updatedAt = event.timestamp
 

src/mappings/shared/metadata.ts

@@ -3,7 +3,7 @@ import { Optional } from '@kodadot1/metasquid/types'
 
 import type { Content } from '@kodadot1/hyperdata'
 import { logger } from '@kodadot1/metasquid/logger'
-import { MetadataEntity as Metadata } from '../../model/generated'
+import { Kind, MetadataEntity as Metadata } from '../../model/generated'
 import { isEmpty } from '../utils/helper'
 import { fetchMetadata } from '../utils/metadata'
 import { Store, attributeFrom } from '../utils/types'
@@ -31,6 +31,7 @@ export async function handleMetadata(id: string, store: Store): Promise<Optional
  name: metadata.name || '',
  type: metadata.type || '',
  banner: metadata.banner || '',
+ kind: metadata.kind as Kind || Kind.mixed,
  }
 
  const final = create<Metadata>(Metadata, id, partial)

src/mappings/uniques/buy.ts

@@ -26,7 +26,7 @@ export async function handleTokenBuy(context: Context): Promise<void> {
  const originalPrice = event.price
  const originalOwner = entity.currentOwner ?? undefined
 
- entity.price = BigInt(0)
+ entity.price = null
  entity.currentOwner = event.caller
  entity.updatedAt = event.timestamp
 

src/mappings/uniques/create.ts

@@ -1,6 +1,6 @@
 import { getOrCreate } from '@kodadot1/metasquid/entity'
 import md5 from 'md5'
-import { CollectionEntity as CE } from '../../model'
+import { CollectionEntity as CE, Kind } from '../../model'
 import { handleMetadata } from '../shared/metadata'
 import { unwrap } from '../utils/extract'
 import { debug, pending, success } from '../utils/logger'
@@ -50,6 +50,7 @@ export async function handleCollectionCreate(context: Context): Promise<void> {
  final.name = metadata?.name
  final.image = metadata?.image
  final.media = metadata?.animationUrl
+ final.kind = metadata?.kind || Kind.mixed
  }
 
  success(OPERATION, `[COLLECTION] ${final.id}`)

src/mappings/uniques/list.ts

@@ -23,7 +23,7 @@ export async function handleTokenList(context: Context): Promise<void> {
  const id = createTokenId(event.collectionId, event.sn)
  const entity = await getWith(context.store, NE, id, { collection: true })
 
- entity.price = event.price
+ entity.price = Boolean(event.price) ? event.price : null
 
  if (event.price && (entity.collection.floor === 0n || event.price < entity.collection.floor)) {
  entity.collection.floor = event.price

src/mappings/uniques/mint.ts

@@ -43,7 +43,7 @@ export async function handleTokenCreate(context: Context): Promise<void> {
  final.collection = collection
  final.sn = BigInt(event.sn)
  final.metadata = event.metadata || collection.metadata
- final.price = BigInt(0)
+ final.price = null
  final.burned = false
  final.createdAt = event.timestamp
  final.updatedAt = event.timestamp

src/mappings/uniques/transfer.ts

@@ -24,7 +24,7 @@ export async function handleTokenTransfer(context: Context): Promise<void> {
  const entity = await getWith(context.store, NE, id, { collection: true })
 
  const oldOwner = entity.currentOwner
- entity.price = BigInt(0)
+ entity.price = null
  entity.currentOwner = event.to
  entity.updatedAt = event.timestamp