Skip to content

Commit

Permalink
Remove last dfract parts. Fixed paginated requests
Browse files Browse the repository at this point in the history
  • Loading branch information
@Segfaultd
Segfaultd committed Feb 14, 2024
1 parent 01895d8 commit e5f5e74
Show file tree
Hide file tree
Showing 17 changed files with 126 additions and 555 deletions.
72 changes: 56 additions & 16 deletions src/async/schedulers/metric.scheduler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import { Pool } from '@lum-network/sdk-javascript/build/codegen/lum/network/mill
import { WithdrawalState } from '@lum-network/sdk-javascript/build/codegen/lum/network/millions/withdrawal';
import { Draw } from '@lum-network/sdk-javascript/build/codegen/lum/network/millions/draw';
import { IdentifiedChannel, State } from '@lum-network/sdk-javascript/build/codegen/ibc/core/channel/v1/channel';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/helpers';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/cosmos/base/query/v1beta1/pagination';

import { AssetSymbol, depositStateToString, MetricNames, sleep, withdrawalStateToString } from '@app/utils';
import { ChainService } from '@app/services';
Expand Down Expand Up @@ -124,13 +124,21 @@ export class MetricScheduler {
this._logger.log(`[MillionsBasics] Syncing...`);

// Acquire list of pools
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const pools: Pool[] = [];
while (true) {
const lPools = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.pools({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const lPools = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.pools({
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(0),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});
pools.push(...lPools.pools);
if (lPools.pagination && lPools.pagination.nextKey && lPools.pagination.nextKey.length > 0) {
page = lPools.pagination.nextKey;
nextPageKey = lPools.pagination.nextKey;
} else {
break;
}
Expand All @@ -144,17 +152,25 @@ export class MetricScheduler {
[DepositState.DEPOSIT_STATE_SUCCESS]: 0,
[DepositState.DEPOSIT_STATE_FAILURE]: 0,
};
page = undefined;
nextPageKey = new Uint8Array();
while (true) {
const deposits = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.deposits({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const deposits = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.deposits({
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(0),
offset: BigInt(0),
reverse: false,
countTotal: false,
})
});

// Increase the given state
for (const deposit of deposits.deposits) {
depositMetas[deposit.state]++;
}

if (deposits.pagination && deposits.pagination.nextKey && deposits.pagination.nextKey.length > 0) {
page = deposits.pagination.nextKey;
nextPageKey = deposits.pagination.nextKey;
} else {
break;
}
Expand All @@ -169,17 +185,25 @@ export class MetricScheduler {
[WithdrawalState.WITHDRAWAL_STATE_PENDING]: 0,
[WithdrawalState.WITHDRAWAL_STATE_FAILURE]: 0,
};
page = undefined;
nextPageKey = new Uint8Array();
while (true) {
const withdrawals = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.withdrawals({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const withdrawals = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.withdrawals({
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(0),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});

// Increase the given state
for (const withdrawal of withdrawals.withdrawals) {
withdrawalMetas[withdrawal.state]++;
}

if (withdrawals.pagination && withdrawals.pagination.nextKey && withdrawals.pagination.nextKey.length > 0) {
page = withdrawals.pagination.nextKey;
nextPageKey = withdrawals.pagination.nextKey;
} else {
break;
}
Expand Down Expand Up @@ -212,13 +236,21 @@ export class MetricScheduler {
this._logger.debug(`[MillionsDraws] Syncing...`);

// Acquire pool draws
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const draws: Draw[] = [];
while (true) {
const lDraws = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.draws({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const lDraws = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).client.lum.network.millions.draws({
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(0),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});
draws.push(...lDraws.draws);
if (lDraws.pagination && lDraws.pagination.nextKey && lDraws.pagination.nextKey.length > 0) {
page = lDraws.pagination.nextKey;
nextPageKey = lDraws.pagination.nextKey;
} else {
break;
}
Expand All @@ -242,13 +274,21 @@ export class MetricScheduler {
this._logger.debug(`[IBC] Syncing...`);

// Grab our channels
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const channels: IdentifiedChannel[] = [];
while (true) {
const lChannels = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).ibcQueryClient.ibc.core.channel.v1.channels({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const lChannels = await this._chainService.getChain<LumChain>(AssetSymbol.LUM).ibcQueryClient.ibc.core.channel.v1.channels({
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(0),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});
channels.push(...lChannels.channels);
if (lChannels.pagination && lChannels.pagination.nextKey && lChannels.pagination.nextKey.length > 0) {
page = lChannels.pagination.nextKey;
nextPageKey = lChannels.pagination.nextKey;
} else {
break;
}
Expand Down
32 changes: 25 additions & 7 deletions src/async/schedulers/millions.scheduler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { Cron, CronExpression } from '@nestjs/schedule';
import dayjs from 'dayjs';
import { convertUnit, LUM_DENOM, MICRO_LUM_DENOM } from '@lum-network/sdk-javascript';
import { Deposit } from '@lum-network/sdk-javascript/build/codegen/lum/network/millions/deposit';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/helpers';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/cosmos/base/query/v1beta1/pagination';

import { MillionsBiggestWinnerEntity, MillionsDepositorEntity, MillionsDrawEntity, MillionsPoolEntity, MillionsPrizeEntity } from '@app/database';
import { ChainService, MarketService, MillionsBiggestWinnerService, MillionsDepositorService, MillionsDrawService, MillionsPoolService, MillionsPrizeService } from '@app/services';
Expand Down Expand Up @@ -134,11 +134,20 @@ export class MillionsScheduler {
} = await lumChain.client.lum.network.millions.params();

for (const pool of pools) {
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();

// Get draws for the pool
while (true) {
const draws = await lumChain.client.lum.network.millions.poolDraws({ poolId: BigInt(pool.id), pagination: page ? ({ key: page } as PageRequest) : undefined });
const draws = await lumChain.client.lum.network.millions.poolDraws({
poolId: BigInt(pool.id),
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(100),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});

for (const draw of draws.draws) {
const id = `${pool.id}-${draw.drawId}`;
Expand Down Expand Up @@ -254,7 +263,7 @@ export class MillionsScheduler {

// If we have pagination key, we just patch it, and it will process in the next loop
if (draws.pagination && draws.pagination.nextKey && draws.pagination.nextKey.length) {
page = draws.pagination.nextKey;
nextPageKey = draws.pagination.nextKey;
} else {
break;
}
Expand All @@ -276,18 +285,27 @@ export class MillionsScheduler {
const lumChain = this._chainService.getChain<LumChain>(AssetSymbol.LUM);

for (const pool of pools) {
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const allDeposits: Deposit[] = [];

// Get all deposits for the pool
while (true) {
const deposits = await lumChain.client.lum.network.millions.poolDeposits({ poolId: BigInt(pool.id), pagination: page ? ({ key: page } as PageRequest) : undefined });
const deposits = await lumChain.client.lum.network.millions.poolDeposits({
poolId: BigInt(pool.id),
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(100),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});

allDeposits.push(...deposits.deposits);

// If we have pagination key, we just patch it, and it will process in the next loop
if (deposits.pagination && deposits.pagination.nextKey && deposits.pagination.nextKey.length) {
page = deposits.pagination.nextKey;
nextPageKey = deposits.pagination.nextKey;
} else {
break;
}
Expand Down
21 changes: 15 additions & 6 deletions src/async/schedulers/validator.scheduler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ import { Cron, CronExpression } from '@nestjs/schedule';
import { ConfigService } from '@nestjs/config';

import { toBech32, LumBech32Prefixes, fromBech32 } from '@lum-network/sdk-javascript';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/helpers';
import { PubKey } from '@lum-network/sdk-javascript/build/codegen/cosmos/crypto/ed25519/keys';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/cosmos/base/query/v1beta1/pagination';

import { ChainService, ValidatorDelegationService, ValidatorService } from '@app/services';
import { ValidatorEntity } from '@app/database';
import { AssetPrefix, AssetSymbol, SIGNED_BLOCK_WINDOW } from '@app/utils';
import { PubKey } from '@lum-network/sdk-javascript/build/codegen/cosmos/crypto/ed25519/keys';

@Injectable()
export class ValidatorScheduler {
Expand Down Expand Up @@ -86,12 +86,21 @@ export class ValidatorScheduler {

// Go through all staking validators to match them with tendermint ones
const statuses = ['BOND_STATUS_BONDED', 'BOND_STATUS_UNBONDED', 'BOND_STATUS_UNBONDING'];
let page: Uint8Array | undefined = undefined;
let nextPageKey: Uint8Array = new Uint8Array();

for (const s of statuses) {
page = undefined;
nextPageKey = new Uint8Array();
while (true) {
const stakingValidators = await this._chainService.getChain(AssetSymbol.LUM).client.cosmos.staking.v1beta1.validators({ status: s as any, pagination: page ? ({ key: page } as PageRequest) : undefined });
const stakingValidators = await this._chainService.getChain(AssetSymbol.LUM).client.cosmos.staking.v1beta1.validators({
status: s as any,
pagination: PageRequest.fromPartial({
key: nextPageKey,
limit: BigInt(100),
offset: BigInt(0),
reverse: false,
countTotal: false,
}),
});

for (const val of stakingValidators.validators) {
const pubKey = PubKey.fromProtoMsg(val.consensusPubkey as any);
Expand Down Expand Up @@ -143,7 +152,7 @@ export class ValidatorScheduler {
}
}
if (stakingValidators.pagination && stakingValidators.pagination.nextKey && stakingValidators.pagination.nextKey.length > 0) {
page = stakingValidators.pagination.nextKey;
nextPageKey = stakingValidators.pagination.nextKey;
} else {
break;
}
Expand Down
55 changes: 0 additions & 55 deletions src/http/controllers/dfract.controller.ts
View file

This file was deleted.

1 change: 0 additions & 1 deletion src/http/controllers/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ export * from './accounts.controller';
export * from './beams.controller';
export * from './blocks.controller';
export * from './core.controller';
export * from './dfract.controller';
export * from './governance.controller';
export * from './health.controller';
export * from './market.controller';
Expand Down
30 changes: 23 additions & 7 deletions src/http/controllers/millions.controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import bcrypt from 'bcrypt';
import { plainToInstance } from 'class-transformer';
import { Withdrawal } from '@lum-network/sdk-javascript/build/codegen/lum/network/millions/withdrawal';
import { Deposit } from '@lum-network/sdk-javascript/build/codegen/lum/network/millions/deposit';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/helpers';
import { PageRequest } from '@lum-network/sdk-javascript/build/codegen/cosmos/base/query/v1beta1/pagination';

import {
DataResponse,
Expand Down Expand Up @@ -315,14 +315,22 @@ export class MillionsController {
const chain = this._chainService.getChain<LumChain>(AssetSymbol.LUM);

// Acquire deposits
let page = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const deposits: Deposit[] = [];
while (true) {
const lDeps = await chain.client.lum.network.millions.deposits({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const lDeps = await chain.client.lum.network.millions.deposits({
pagination: PageRequest.fromPartial({
key: nextPageKey,
offset: BigInt(0),
limit: BigInt(100),
reverse: false,
countTotal: false,
}),
});
deposits.push(...lDeps.deposits);

if (lDeps.pagination && lDeps.pagination.nextKey && lDeps.pagination.nextKey.length > 0) {
page = lDeps.pagination.nextKey;
nextPageKey = lDeps.pagination.nextKey;
} else {
break;
}
Expand All @@ -344,14 +352,22 @@ export class MillionsController {
const chain = this._chainService.getChain<LumChain>(AssetSymbol.LUM);

// Acquire deposits
let page = undefined;
let nextPageKey: Uint8Array = new Uint8Array();
const withdrawals: Withdrawal[] = [];
while (true) {
const lWdls = await chain.client.lum.network.millions.withdrawals({ pagination: page ? ({ key: page } as PageRequest) : undefined });
const lWdls = await chain.client.lum.network.millions.withdrawals({
pagination: PageRequest.fromPartial({
key: nextPageKey,
offset: BigInt(0),
limit: BigInt(100),
reverse: false,
countTotal: false,
}),
});
withdrawals.push(...lWdls.withdrawals);

if (lWdls.pagination && lWdls.pagination.nextKey && lWdls.pagination.nextKey.length > 0) {
page = lWdls.pagination.nextKey;
nextPageKey = lWdls.pagination.nextKey;
} else {
break;
}
Expand Down
Loading

0 comments on commit e5f5e74

Please sign in to comment.