Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

remove support for indexer v 3 #1324

Merged
merged 5 commits into from
Sep 25, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion config/config.devnet-old.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ flags:
useRequestLogging: false
useVmQueryTracing: false
processNfts: true
indexer-v3: true
collectionPropertiesFromGateway: false
features:
eventsNotifier:
Expand Down
3 changes: 1 addition & 2 deletions config/config.devnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ flags:
useRequestLogging: false
useVmQueryTracing: false
processNfts: true
indexer-v3: true
collectionPropertiesFromGateway: false
features:
eventsNotifier:
Expand Down Expand Up @@ -165,4 +164,4 @@ inflation:
- 719203
nftProcess:
parallelism: 1
maxRetries: 3
maxRetries: 3
1 change: 0 additions & 1 deletion config/config.e2e-mocked.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ flags:
useRequestCaching: true
useKeepAliveAgent: true
useTracing: false
indexer-v3: true
collectionPropertiesFromGateway: false
urls:
self: 'https://api.multiversx.com'
Expand Down
1 change: 0 additions & 1 deletion config/config.e2e.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ flags:
useRequestCaching: true
useKeepAliveAgent: true
useTracing: false
indexer-v3: true
collectionPropertiesFromGateway: false
urls:
self: 'https://api.multiversx.com'
Expand Down
3 changes: 1 addition & 2 deletions config/config.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ flags:
useRequestLogging: false
useVmQueryTracing: false
processNfts: true
indexer-v3: false
collectionPropertiesFromGateway: false
features:
eventsNotifier:
Expand Down Expand Up @@ -169,4 +168,4 @@ inflation:
- 719203
nftProcess:
parallelism: 1
maxRetries: 3
maxRetries: 3
1 change: 0 additions & 1 deletion config/config.testnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ flags:
useRequestLogging: false
useVmQueryTracing: false
processNfts: true
indexer-v3: true
collectionPropertiesFromGateway: false
features:
eventsNotifier:
Expand Down
4 changes: 0 additions & 4 deletions src/common/api-config/api.config.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -367,10 +367,6 @@ export class ApiConfigService {
return this.configService.get<boolean>('features.processNfts.enabled') ?? this.configService.get<boolean>('flags.processNfts') ?? false;
}

getIsIndexerV3FlagActive(): boolean {
return this.configService.get<boolean>('flags.indexer-v3') ?? false;
}

isGraphQlActive(): boolean {
return this.configService.get<boolean>('api.graphql') ?? false;
}
Expand Down
92 changes: 36 additions & 56 deletions src/common/indexer/elastic/elastic.indexer.helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -83,51 +83,45 @@ export class ElasticIndexerHelper {
.withMustMultiShouldCondition(Object.values(NftType), type => QueryType.Match('type', type));

if (address) {
if (this.apiConfigService.getIsIndexerV3FlagActive()) {
elasticQuery = elasticQuery.withMustCondition(QueryType.Should(
[
QueryType.Match('currentOwner', address),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTCreate', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTBurn', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTAddQuantity', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTUpdateAttributes', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTAddURI', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTTransferRole', address)]),
]
));
} else {
elasticQuery = elasticQuery.withMustCondition(QueryType.Match('currentOwner', address));
}
elasticQuery = elasticQuery.withMustCondition(QueryType.Should(
[
QueryType.Match('currentOwner', address),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTCreate', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTBurn', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTAddQuantity', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTUpdateAttributes', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTRoleNFTAddURI', address)]),
QueryType.Nested('roles', [new MatchQuery('roles.ESDTTransferRole', address)]),
]
));
}

if (filter.before || filter.after) {
elasticQuery = elasticQuery.withDateRangeFilter('timestamp', filter.before, filter.after);
}

if (this.apiConfigService.getIsIndexerV3FlagActive()) {
if (filter.canCreate !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTCreate', address, filter.canCreate);
}
if (filter.canCreate !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTCreate', address, filter.canCreate);
}

if (filter.canBurn !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTBurn', address, filter.canBurn);
}
if (filter.canBurn !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTBurn', address, filter.canBurn);
}

if (filter.canAddQuantity !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTAddQuantity', address, filter.canAddQuantity);
}
if (filter.canAddQuantity !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTAddQuantity', address, filter.canAddQuantity);
}

if (filter.canUpdateAttributes !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTUpdateAttributes', address, filter.canUpdateAttributes);
}
if (filter.canUpdateAttributes !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTUpdateAttributes', address, filter.canUpdateAttributes);
}

if (filter.canAddUri !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTAddURI', address, filter.canAddUri);
}
if (filter.canAddUri !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTRoleNFTAddURI', address, filter.canAddUri);
}

if (filter.canTransferRole !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTTransferRole', address, filter.canTransferRole);
}
if (filter.canTransferRole !== undefined) {
elasticQuery = this.getRoleCondition(elasticQuery, 'ESDTTransferRole', address, filter.canTransferRole);
}

if (filter.excludeMetaESDT === true) {
Expand Down Expand Up @@ -219,7 +213,7 @@ export class ElasticIndexerHelper {
elasticQuery = elasticQuery.withMustCondition(QueryType.Should(filter.identifiers.map(identifier => QueryType.Match('identifier', identifier, QueryOperator.AND))));
}

if (filter.isWhitelistedStorage !== undefined && this.apiConfigService.getIsIndexerV3FlagActive()) {
if (filter.isWhitelistedStorage !== undefined) {
elasticQuery = elasticQuery.withMustCondition(QueryType.Nested("data", [new MatchQuery("data.whiteListedStorage", filter.isWhitelistedStorage)]));
}

Expand Down Expand Up @@ -329,7 +323,7 @@ export class ElasticIndexerHelper {
elasticQuery = elasticQuery.withMustMultiShouldCondition(filter.tokens, token => QueryType.Match('tokens', token, QueryOperator.AND));
}

if (filter.functions && filter.functions.length > 0 && this.apiConfigService.getIsIndexerV3FlagActive()) {
if (filter.functions && filter.functions.length > 0) {
if (filter.functions.length === 1 && filter.functions[0] === '') {
elasticQuery = elasticQuery.withMustNotExistCondition('function');
} else {
Expand Down Expand Up @@ -459,10 +453,7 @@ export class ElasticIndexerHelper {
if (address) {
shouldQueries.push(QueryType.Match('sender', address));
shouldQueries.push(QueryType.Match('receiver', address));

if (this.apiConfigService.getIsIndexerV3FlagActive()) {
shouldQueries.push(QueryType.Match('receivers', address));
}
shouldQueries.push(QueryType.Match('receivers', address));
}

const elasticQuery = ElasticQuery.create()
Expand All @@ -484,7 +475,7 @@ export class ElasticIndexerHelper {
.withMustMultiShouldCondition(filter.tokens, token => QueryType.Match('tokens', token, QueryOperator.AND))
.withDateRangeFilter('timestamp', filter.before, filter.after);

if (filter.functions && filter.functions.length > 0 && this.apiConfigService.getIsIndexerV3FlagActive()) {
if (filter.functions && filter.functions.length > 0) {
if (filter.functions.length === 1 && filter.functions[0] === '') {
elasticQuery = elasticQuery.withMustNotExistCondition('function');
} else {
Expand All @@ -508,10 +499,7 @@ export class ElasticIndexerHelper {
}

if (filter.receivers) {
const keys = ['receiver'];
if (this.apiConfigService.getIsIndexerV3FlagActive()) {
keys.push('receivers');
}
const keys = ['receiver', 'receivers'];

for (const receiver of filter.receivers) {
for (const key of keys) {
Expand All @@ -523,11 +511,7 @@ export class ElasticIndexerHelper {
elasticQuery = elasticQuery.withMustMatchCondition('sender', filter.sender);

if (filter.receivers) {
const keys = ['receiver'];

if (this.apiConfigService.getIsIndexerV3FlagActive()) {
keys.push('receivers');
}
const keys = ['receiver', 'receivers'];

const queries: AbstractQuery[] = [];

Expand All @@ -542,11 +526,7 @@ export class ElasticIndexerHelper {
}

if (address) {
const keys: string[] = ['sender', 'receiver'];

if (this.apiConfigService.getIsIndexerV3FlagActive()) {
keys.push('receivers');
}
const keys: string[] = ['sender', 'receiver', 'receivers'];

elasticQuery = elasticQuery.withMustMultiShouldCondition(keys, key => QueryType.Match(key, address));
}
Expand Down Expand Up @@ -655,7 +635,7 @@ export class ElasticIndexerHelper {
elasticQuery = elasticQuery.withShouldCondition(QueryType.Match('receiver', filter.receiver));
}

if (filter.functions && filter.functions.length > 0 && this.apiConfigService.getIsIndexerV3FlagActive()) {
if (filter.functions && filter.functions.length > 0) {
if (filter.functions.length === 1 && filter.functions[0] === '') {
elasticQuery = elasticQuery.withMustNotExistCondition('function');
} else {
Expand Down
33 changes: 4 additions & 29 deletions src/common/indexer/elastic/elastic.indexer.service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { HttpStatus, Injectable } from "@nestjs/common";
import { BinaryUtils } from "@multiversx/sdk-nestjs-common";
import { ApiService } from "@multiversx/sdk-nestjs-http";
import { ElasticService, ElasticQuery, QueryOperator, QueryType, QueryConditionOptions, ElasticSortOrder, ElasticSortProperty, TermsQuery, RangeGreaterThanOrEqual, MatchQuery } from "@multiversx/sdk-nestjs-elastic";
import { IndexerInterface } from "../indexer.interface";
import { ApiConfigService } from "src/common/api-config/api.config.service";
Expand Down Expand Up @@ -36,7 +35,6 @@ export class ElasticIndexerService implements IndexerInterface {
private readonly apiConfigService: ApiConfigService,
private readonly elasticService: ElasticService,
private readonly indexerHelper: ElasticIndexerHelper,
private readonly apiService: ApiService,
) { }

async getAccountsCount(filter: AccountQueryOptions): Promise<number> {
Expand Down Expand Up @@ -653,17 +651,12 @@ export class ElasticIndexerService implements IndexerInterface {
}

async getNftsForAddress(address: string, filter: NftFilter, pagination: QueryPagination): Promise<any[]> {
let elasticQuery = this.indexerHelper.buildElasticNftFilter(filter, undefined, address)
.withPagination(pagination);

if (this.apiConfigService.getIsIndexerV3FlagActive()) {
elasticQuery = elasticQuery.withSort([
const elasticQuery = this.indexerHelper.buildElasticNftFilter(filter, undefined, address)
.withPagination(pagination)
.withSort([
{ name: 'timestamp', order: ElasticSortOrder.descending },
{ name: 'tokenNonce', order: ElasticSortOrder.descending },
]);
} else {
elasticQuery = elasticQuery.withSort([{ name: '_id', order: ElasticSortOrder.ascending }]);
}

return await this.elasticService.getList('accountsesdt', 'identifier', elasticQuery);
}
Expand Down Expand Up @@ -760,22 +753,6 @@ export class ElasticIndexerService implements IndexerInterface {
return undefined;
}

private indexerV5Active: boolean | undefined = undefined;

async isIndexerV5Active(): Promise<boolean> {
if (this.indexerV5Active !== undefined) {
return this.indexerV5Active;
}

const mappingsResult = await this.apiService.get(`${this.apiConfigService.getElasticUrl()}/tokens/_mappings`);
const mappings = mappingsResult.data?.tokens?.mappings?.properties ?? mappingsResult.data['tokens-000001']?.mappings?.properties;

const currentOwnerType = mappings?.currentOwner?.type;

this.indexerV5Active = currentOwnerType === 'keyword';
return this.indexerV5Active;
}

async getCollectionsForAddress(
address: string,
filter: CollectionFilter,
Expand All @@ -786,8 +763,6 @@ export class ElasticIndexerService implements IndexerInterface {
types.push(NftType.MetaESDT);
}

const isIndexerV5Active = await this.isIndexerV5Active();

const elasticQuery = ElasticQuery.create()
.withMustExistCondition('identifier')
.withMustMatchCondition('address', address)
Expand All @@ -806,7 +781,7 @@ export class ElasticIndexerService implements IndexerInterface {
{
collection: {
terms: {
field: isIndexerV5Active ? 'token' : 'token.keyword',
field: 'token.keyword',
},
},
},
Expand Down
14 changes: 0 additions & 14 deletions src/endpoints/accounts/account.controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@ import { AccountHistory } from "./entities/account.history";
import { AccountEsdtHistory } from "./entities/account.esdt.history";
import { EsdtDataSource } from '../esdt/entities/esdt.data.source';
import { TransferService } from '../transfers/transfer.service';
import { ApiConfigService } from 'src/common/api-config/api.config.service';
import { Transaction } from '../transactions/entities/transaction';
import { ProviderStake } from '../stake/entities/provider.stake';
import { TokenDetailedWithBalance } from '../tokens/entities/token.detailed.with.balance';
Expand Down Expand Up @@ -74,7 +73,6 @@ export class AccountController {
private readonly scResultService: SmartContractResultService,
private readonly collectionService: CollectionService,
private readonly transferService: TransferService,
private readonly apiConfigService: ApiConfigService,
private readonly delegationService: DelegationService,
) { }

Expand Down Expand Up @@ -973,10 +971,6 @@ export class AccountController {
@Query('withOperations', new ParseBoolPipe) withOperations?: boolean,
@Query('withActionTransferValue', ParseBoolPipe) withActionTransferValue?: boolean,
): Promise<Transaction[]> {
if (!this.apiConfigService.getIsIndexerV3FlagActive()) {
throw new HttpException('Endpoint not live yet', HttpStatus.NOT_IMPLEMENTED);
}

const options = TransactionQueryOptions.applyDefaultOptions(
size, { withScamInfo, withUsername, withBlockInfo, withOperations, withLogs, withActionTransferValue });

Expand Down Expand Up @@ -1032,10 +1026,6 @@ export class AccountController {
@Query('after', ParseIntPipe) after?: number,
@Query('senderOrReceiver', ParseAddressPipe) senderOrReceiver?: string,
): Promise<number> {
if (!this.apiConfigService.getIsIndexerV3FlagActive()) {
throw new HttpException('Endpoint not live yet', HttpStatus.NOT_IMPLEMENTED);
}

return await this.transferService.getTransfersCount(new TransactionFilter({
address,
senders: sender,
Expand Down Expand Up @@ -1070,10 +1060,6 @@ export class AccountController {
@Query('after', ParseIntPipe) after?: number,
@Query('senderOrReceiver', ParseAddressPipe) senderOrReceiver?: string,
): Promise<number> {
if (!this.apiConfigService.getIsIndexerV3FlagActive()) {
throw new HttpException('Endpoint not live yet', HttpStatus.NOT_IMPLEMENTED);
}

return await this.transferService.getTransfersCount(new TransactionFilter({
address,
senders: sender,
Expand Down
Loading
Loading