From fed374fbbbfca03f0b8d5aaad1d02d6219d3ef34 Mon Sep 17 00:00:00 2001 From: Gheorghe Pinzaru Date: Thu, 28 Nov 2024 21:01:36 +0700 Subject: [PATCH 1/2] Use TaggedRequest for request to allow for equality check in cache --- .changeset/green-elephants-guess.md | 5 +++ .../transaction-decoder/src/abi-loader.ts | 38 +++++++++++++++---- .../src/abi-strategy/request-model.ts | 27 ++++++++----- .../src/contract-meta-loader.ts | 26 ++++++++----- .../src/meta-strategy/request-model.ts | 25 ++++++++---- 5 files changed, 86 insertions(+), 35 deletions(-) create mode 100644 .changeset/green-elephants-guess.md diff --git a/.changeset/green-elephants-guess.md b/.changeset/green-elephants-guess.md new file mode 100644 index 0000000..a48ecbb --- /dev/null +++ b/.changeset/green-elephants-guess.md @@ -0,0 +1,5 @@ +--- +'@3loop/transaction-decoder': patch +--- + +Use TaggedRequest for request to allow for equality check for in-memory caching of requests diff --git a/packages/transaction-decoder/src/abi-loader.ts b/packages/transaction-decoder/src/abi-loader.ts index 4a94cb4..d1eeb44 100644 --- a/packages/transaction-decoder/src/abi-loader.ts +++ b/packages/transaction-decoder/src/abi-loader.ts @@ -1,4 +1,16 @@ -import { Context, Effect, Either, RequestResolver, Request, Array, pipe, Data } from 'effect' +import { + Context, + Effect, + Either, + RequestResolver, + Request, + Array, + pipe, + Data, + PrimaryKey, + Schema, + SchemaAST, +} from 'effect' import { ContractABI, ContractAbiResolverStrategy, GetContractABIStrategy } from './abi-strategy/request-model.js' import { Abi } from 'viem' @@ -63,12 +75,22 @@ export class EmptyCalldataError extends Data.TaggedError('DecodeError')< } } -export interface AbiLoader extends Request.Request, LoadParameters { - _tag: 'AbiLoader' +class SchemaAbi extends Schema.make(SchemaAST.objectKeyword) {} +class AbiLoader extends Schema.TaggedRequest()('AbiLoader', { + failure: Schema.instanceOf(MissingABIError), + success: SchemaAbi, // Abi + payload: { + chainID: Schema.Number, + address: Schema.String, + event: Schema.optional(Schema.String), + signature: Schema.optional(Schema.String), + }, +}) { + [PrimaryKey.symbol]() { + return `abi::${this.chainID}:${this.address}:${this.event}:${this.signature}` + } } -const AbiLoader = Request.tagged('AbiLoader') - function makeRequestKey(key: AbiLoader) { return `abi::${key.chainID}:${key.address}:${key.event}:${key.signature}` } @@ -191,7 +213,7 @@ const AbiLoaderRequestResolver: Effect.Effect< // NOTE: Firstly we batch strategies by address because in a transaction most of events and traces are from the same abi const response = yield* Effect.forEach(remaining, (req) => { - const strategyRequest = GetContractABIStrategy({ + const strategyRequest = new GetContractABIStrategy({ address: req.address, chainID: req.chainID, }) @@ -216,7 +238,7 @@ const AbiLoaderRequestResolver: Effect.Effect< // NOTE: Secondly we request strategies to fetch fragments const fragmentStrategyResults = yield* Effect.forEach(notFound, ({ chainID, address, event, signature }) => { - const strategyRequest = GetContractABIStrategy({ + const strategyRequest = new GetContractABIStrategy({ address, chainID, event, @@ -271,7 +293,7 @@ export const getAndCacheAbi = (params: AbiParams) => return yield* Effect.fail(new EmptyCalldataError(params)) } - return yield* Effect.request(AbiLoader(params), AbiLoaderRequestResolver) + return yield* Effect.request(new AbiLoader(params), AbiLoaderRequestResolver) }).pipe( Effect.withSpan('AbiLoader.GetAndCacheAbi', { attributes: { diff --git a/packages/transaction-decoder/src/abi-strategy/request-model.ts b/packages/transaction-decoder/src/abi-strategy/request-model.ts index ae237d5..610d806 100644 --- a/packages/transaction-decoder/src/abi-strategy/request-model.ts +++ b/packages/transaction-decoder/src/abi-strategy/request-model.ts @@ -1,4 +1,4 @@ -import { Request, RequestResolver } from 'effect' +import { PrimaryKey, RequestResolver, Schema, SchemaAST } from 'effect' export interface FetchABIParams { readonly chainID: number @@ -41,16 +41,23 @@ interface AddressABI { export type ContractABI = FunctionFragmentABI | EventFragmentABI | AddressABI -// NOTE: We might want to return a list of ABIs, this might be helpful when fetching for signature -export interface GetContractABIStrategy - extends Request.Request, - FetchABIParams { - readonly _tag: 'GetContractABIStrategy' -} - -export const GetContractABIStrategy = Request.tagged('GetContractABIStrategy') - export interface ContractAbiResolverStrategy { type: 'address' | 'fragment' resolver: RequestResolver.RequestResolver } + +class SchemaContractAbi extends Schema.make(SchemaAST.objectKeyword) {} +export class GetContractABIStrategy extends Schema.TaggedRequest()('GetContractABIStrategy', { + failure: Schema.instanceOf(ResolveStrategyABIError), + success: Schema.Array(SchemaContractAbi), + payload: { + chainID: Schema.Number, + address: Schema.String, + event: Schema.optional(Schema.String), + signature: Schema.optional(Schema.String), + }, +}) { + [PrimaryKey.symbol]() { + return `abi-strategy::${this.chainID}:${this.address}:${this.event}:${this.signature}` + } +} diff --git a/packages/transaction-decoder/src/contract-meta-loader.ts b/packages/transaction-decoder/src/contract-meta-loader.ts index f398a96..0046061 100644 --- a/packages/transaction-decoder/src/contract-meta-loader.ts +++ b/packages/transaction-decoder/src/contract-meta-loader.ts @@ -1,4 +1,4 @@ -import { Context, Effect, RequestResolver, Request, Array, Either, pipe } from 'effect' +import { Context, Effect, RequestResolver, Request, Array, Either, pipe, Schema, PrimaryKey, SchemaAST } from 'effect' import { ContractData } from './types.js' import { GetContractMetaStrategy } from './meta-strategy/request-model.js' import { Address } from 'viem' @@ -50,14 +50,22 @@ export interface ContractMetaStore('@3loop-decoder/ContractMetaStore') -export interface ContractMetaLoader extends Request.Request { - _tag: 'ContractMetaLoader' - address: Address - chainID: number +class SchemaContractData extends Schema.make(SchemaAST.objectKeyword) {} +class SchemaAddress extends Schema.make
(SchemaAST.stringKeyword) {} + +class ContractMetaLoader extends Schema.TaggedRequest()('ContractMetaLoader', { + failure: Schema.Never, + success: Schema.NullOr(SchemaContractData), + payload: { + address: SchemaAddress, + chainID: Schema.Number, + }, +}) { + [PrimaryKey.symbol]() { + return `contract-meta::${this.chainID}:${this.address}` + } } -const ContractMetaLoader = Request.tagged('ContractMetaLoader') - function makeKey(key: ContractMetaLoader) { return `contract-meta::${key.chainID}:${key.address}` } @@ -148,7 +156,7 @@ const ContractMetaLoaderRequestResolver = RequestResolver.makeBatched((requests: // Fetch ContractMeta from the strategies const strategyResults = yield* Effect.forEach(remaining, ({ chainID, address }) => { - const strategyRequest = GetContractMetaStrategy({ + const strategyRequest = new GetContractMetaStrategy({ address, chainID, }) @@ -186,7 +194,7 @@ export const getAndCacheContractMeta = ({ readonly address: Address }) => { return Effect.withSpan( - Effect.request(ContractMetaLoader({ chainID, address }), ContractMetaLoaderRequestResolver), + Effect.request(new ContractMetaLoader({ chainID, address }), ContractMetaLoaderRequestResolver), 'GetAndCacheContractMeta', { attributes: { chainID, address } }, ) diff --git a/packages/transaction-decoder/src/meta-strategy/request-model.ts b/packages/transaction-decoder/src/meta-strategy/request-model.ts index 3586584..fe10cf9 100644 --- a/packages/transaction-decoder/src/meta-strategy/request-model.ts +++ b/packages/transaction-decoder/src/meta-strategy/request-model.ts @@ -1,6 +1,6 @@ import { UnknownNetwork } from '../public-client.js' import { ContractData } from '../types.js' -import { Request } from 'effect' +import { PrimaryKey, Schema, SchemaAST } from 'effect' import { Address } from 'viem' export interface FetchMetaParams { @@ -17,11 +17,20 @@ export class ResolveStrategyMetaError { ) {} } -// TODO: Remove UnknownNetwork -export interface GetContractMetaStrategy - extends Request.Request, - FetchMetaParams { - readonly _tag: 'GetContractMetaStrategy' +class SchemaAddress extends Schema.make
(SchemaAST.stringKeyword) {} +class SchemaContractData extends Schema.make(SchemaAST.objectKeyword) {} +export class GetContractMetaStrategy extends Schema.TaggedRequest()( + 'GetContractMetaStrategy', + { + failure: Schema.Union(Schema.instanceOf(ResolveStrategyMetaError), Schema.instanceOf(UnknownNetwork)), + success: SchemaContractData, + payload: { + chainID: Schema.Number, + address: SchemaAddress, + }, + }, +) { + [PrimaryKey.symbol]() { + return `contract-meta-strategy::${this.chainID}:${this.address}` + } } - -export const GetContractMetaStrategy = Request.tagged('GetContractMetaStrategy') From ae7b1baea3c90c63add4c466179fb225dcd456e5 Mon Sep 17 00:00:00 2001 From: Gheorghe Pinzaru Date: Thu, 28 Nov 2024 23:25:03 +0700 Subject: [PATCH 2/2] Fix sql stores syntax --- .changeset/wet-mirrors-work.md | 5 ++ .../transaction-decoder/src/sql/abi-store.ts | 77 +++++++++++++++---- .../src/sql/contract-meta-store.ts | 69 ++++++++++++----- 3 files changed, 114 insertions(+), 37 deletions(-) create mode 100644 .changeset/wet-mirrors-work.md diff --git a/.changeset/wet-mirrors-work.md b/.changeset/wet-mirrors-work.md new file mode 100644 index 0000000..3709fee --- /dev/null +++ b/.changeset/wet-mirrors-work.md @@ -0,0 +1,5 @@ +--- +'@3loop/transaction-decoder': minor +--- + +Fix sql stores syntax diff --git a/packages/transaction-decoder/src/sql/abi-store.ts b/packages/transaction-decoder/src/sql/abi-store.ts index 64cf76e..edbcd08 100644 --- a/packages/transaction-decoder/src/sql/abi-store.ts +++ b/packages/transaction-decoder/src/sql/abi-store.ts @@ -8,8 +8,11 @@ export const make = (strategies: AbiStore['strategies']) => Effect.gen(function* () { const sql = yield* SqlClient.SqlClient + const table = sql('loop_decoder_contract_abi__') + + // TODO; add timestamp to the table yield* sql` - CREATE TABLE IF NOT EXISTS contractAbi ( + CREATE TABLE IF NOT EXISTS ${table} ( type TEXT NOT NULL, address TEXT, event TEXT, @@ -18,7 +21,10 @@ export const make = (strategies: AbiStore['strategies']) => abi TEXT, status TEXT NOT NULL ) - `.pipe(Effect.catchAll(() => Effect.dieMessage('Failed to create contractAbi table'))) + `.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.dieMessage('Failed to create contractAbi table')), + ) return AbiStore.of({ strategies, @@ -28,33 +34,70 @@ export const make = (strategies: AbiStore['strategies']) => if (value.status === 'success' && value.result.type === 'address') { const result = value.result yield* sql` - INSERT INTO contractAbi (type, address, chain, abi, status) - VALUES (${result.type}, ${normalizedAddress}, ${result.chainID}, ${result.abi}, "success") + INSERT INTO ${table} + ${sql.insert([ + { + type: result.type, + address: normalizedAddress, + chain: key.chainID, + abi: result.abi, + status: 'success', + }, + ])} ` } else if (value.status === 'success') { const result = value.result yield* sql` - INSERT INTO contractAbi (type, event, signature, abi, status) - VALUES (${result.type}, ${'event' in result ? result.event : null}, ${ - 'signature' in result ? result.signature : null - }, ${result.abi}, "success") + INSERT INTO ${table} + ${sql.insert([ + { + type: result.type, + event: 'event' in result ? result.event : null, + signature: 'signature' in result ? result.signature : null, + abi: result.abi, + status: 'success', + }, + ])} ` } else { yield* sql` - INSERT INTO contractAbi (type, address, chain, status) - VALUES ("address", ${normalizedAddress}, ${key.chainID}, "not-found") + INSERT INTO ${table} + ${sql.insert([ + { + type: 'address', + address: normalizedAddress, + chain: key.chainID, + status: 'not-found', + }, + ])} ` } - }).pipe(Effect.catchAll(() => Effect.succeed(null))), + }).pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => { + return Effect.succeed(null) + }), + ), get: ({ address, signature, event, chainID }) => Effect.gen(function* () { - const items = yield* sql` - SELECT * FROM contractAbi - WHERE (address = ${address.toLowerCase()} AND chain = ${chainID} AND type = "address") - ${signature ? `OR (signature = ${signature} AND type = "func")` : ''} - ${event ? `OR (event = ${event} AND type = "event")` : ''} - `.pipe(Effect.catchAll(() => Effect.succeed([]))) + const addressQuery = sql.and([ + sql`address = ${address.toLowerCase()}`, + sql`chain = ${chainID}`, + sql`type = 'address'`, + ]) + + const signatureQuery = signature ? sql.and([sql`signature = ${signature}`, sql`type = 'func'`]) : undefined + const eventQuery = event ? sql.and([sql`event = ${event}`, sql`type = 'event'`]) : undefined + const query = + signature == null && event == null + ? addressQuery + : sql.or([addressQuery, signatureQuery, eventQuery].filter(Boolean)) + + const items = yield* sql` SELECT * FROM ${table} WHERE ${query}`.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.succeed([])), + ) const item = items.find((item) => { diff --git a/packages/transaction-decoder/src/sql/contract-meta-store.ts b/packages/transaction-decoder/src/sql/contract-meta-store.ts index 5078f12..063e8e8 100644 --- a/packages/transaction-decoder/src/sql/contract-meta-store.ts +++ b/packages/transaction-decoder/src/sql/contract-meta-store.ts @@ -16,17 +16,23 @@ export const make = () => const sql = yield* SqlClient.SqlClient const publicClient = yield* PublicClient + const table = sql('loop_decoder_contract_meta__') + + // TODO; add timestamp to the table yield* sql` - CREATE TABLE IF NOT EXISTS contractMeta ( + CREATE TABLE IF NOT EXISTS ${table} ( address TEXT NOT NULL, chain INTEGER NOT NULL, - contractName TEXT, - tokenSymbol TEXT, + contract_name TEXT, + token_symbol TEXT, decimals INTEGER, type TEXT, status TEXT NOT NULL ) - `.pipe(Effect.catchAll(() => Effect.dieMessage('Failed to create contractMeta table'))) + `.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.dieMessage('Failed to create contractMeta table')), + ) return ContractMetaStore.of({ strategies: { @@ -39,28 +45,51 @@ export const make = () => set: (key, value) => Effect.gen(function* () { if (value.status === 'success') { - const name = value.result.contractName ?? null - const symbol = value.result.tokenSymbol ?? null - const decimals = value.result.decimals ?? null + const name = value.result.contractName ?? '' + const symbol = value.result.tokenSymbol ?? '' + const decimals = value.result.decimals ?? undefined + + const clear = Object.fromEntries( + Object.entries({ + address: key.address.toLowerCase(), + chain: key.chainID, + contract_name: name, + token_symbol: symbol, + decimals, + type: value.result.type, + status: 'success', + }).filter(([_, v]) => v !== undefined), + ) yield* sql` - INSERT INTO contractMeta (address, chain, contractName, tokenSymbol, decimals, type, status) - VALUES (${key.address.toLowerCase()}, ${key.chainID}, ${name}, ${symbol}, ${decimals}, - ${value.result.type}, "success") - ` + INSERT INTO ${table} + ${sql.insert([clear])} + ` } else { yield* sql` - INSERT INTO contractMeta (address, chain, contractName, tokenSymbol, decimals, type, status) - VALUES (${key.address.toLowerCase()}, ${key.chainID}, null, null, null, null, "not-found") - ` + INSERT INTO ${table} + ${sql.insert([ + { + address: key.address.toLowerCase(), + chain: key.chainID, + status: 'not-found', + }, + ])} + ` } - }).pipe(Effect.catchAll(() => Effect.succeed(null))), + }).pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.succeed(null)), + ), get: ({ address, chainID }) => Effect.gen(function* () { const items = yield* sql` - SELECT * FROM contractMeta - WHERE address = ${address.toLowerCase()} AND chain = ${chainID} - `.pipe(Effect.catchAll(() => Effect.succeed([]))) + SELECT * FROM ${table} + WHERE ${sql.and([sql`address = ${address.toLowerCase()}`, sql`chain = ${chainID}`])} + `.pipe( + Effect.tapError(Effect.logError), + Effect.catchAll(() => Effect.succeed([])), + ) const item = items[0] @@ -69,8 +98,8 @@ export const make = () => status: 'success', result: { contractAddress: address, - contractName: item.contractName, - tokenSymbol: item.tokenSymbol, + contractName: item.contract_name, + tokenSymbol: item.token_symbol, decimals: item.decimals, type: item.type, address,