Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: pagination returns less than the page size when decryption fails #1547

Merged
merged 22 commits into from
Jan 22, 2025
Merged
Changes from 20 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions packages/data-access/src/combined-data-access.ts
Original file line number Diff line number Diff line change
@@ -31,18 +31,14 @@ export abstract class CombinedDataAccess implements DataAccessTypes.IDataAccess
async getChannelsByTopic(
topic: string,
updatedBetween?: DataAccessTypes.ITimestampBoundaries | undefined,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
return await this.reader.getChannelsByTopic(topic, updatedBetween, page, pageSize);
return await this.reader.getChannelsByTopic(topic, updatedBetween);
}
async getChannelsByMultipleTopics(
topics: string[],
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number | undefined,
pageSize?: number | undefined,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween, page, pageSize);
return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween);
}
async persistTransaction(
transactionData: DataAccessTypes.ITransaction,
53 changes: 2 additions & 51 deletions packages/data-access/src/data-read.ts
Original file line number Diff line number Diff line change
@@ -49,26 +49,15 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
async getChannelsByTopic(
topic: string,
updatedBetween?: DataAccessTypes.ITimestampBoundaries | undefined,
page?: number | undefined,
pageSize?: number | undefined,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
return this.getChannelsByMultipleTopics([topic], updatedBetween, page, pageSize);
return this.getChannelsByMultipleTopics([topic], updatedBetween);
}

async getChannelsByMultipleTopics(
topics: string[],
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
// Validate pagination parameters
if (page !== undefined && page < 1) {
throw new Error(`Page number must be greater than or equal to 1, but it is ${page}`);
}
if (pageSize !== undefined && pageSize < 1) {
throw new Error(`Page size must be greater than 0, but it is ${pageSize}`);
}

const result = await this.storage.getTransactionsByTopics(topics);
const pending = this.pendingStore?.findByTopics(topics) || [];

const pendingItems = pending.map((item) => ({
@@ -84,33 +73,6 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
topics: item.topics || [],
}));

// Calculate adjusted pagination
let adjustedPage = page;
let adjustedPageSize = pageSize;
let pendingItemsOnCurrentPage = 0;
if (page !== undefined && pageSize !== undefined) {
const totalPending = pendingItems.length;
const itemsPerPage = (page - 1) * pageSize;

if (totalPending > itemsPerPage) {
pendingItemsOnCurrentPage = Math.min(totalPending - itemsPerPage, pageSize);
adjustedPageSize = pageSize - pendingItemsOnCurrentPage;
adjustedPage = 1;
if (adjustedPageSize === 0) {
adjustedPageSize = 1;
pendingItemsOnCurrentPage--;
}
} else {
adjustedPage = page - Math.floor(totalPending / pageSize);
}
}

const result = await this.storage.getTransactionsByTopics(
topics,
adjustedPage,
adjustedPageSize,
);

const transactions = result.transactions.concat(...pendingItems);

// list of channels having at least one tx updated during the updatedBetween boundaries
@@ -138,17 +100,6 @@ export class DataAccessRead implements DataAccessTypes.IDataRead {
prev[curr.channelId].push(curr.hash);
return prev;
}, {} as Record<string, string[]>),
pagination:
page && pageSize
? {
total: result.transactions.length + pendingItems.length,
page,
pageSize,
hasMore:
(page - 1) * pageSize + filteredTxs.length - pendingItemsOnCurrentPage <
result.transactions.length,
}
: undefined,
},
result: {
transactions: filteredTxs.reduce((prev, curr) => {
31 changes: 2 additions & 29 deletions packages/data-access/src/in-memory-indexer.ts
Original file line number Diff line number Diff line change
@@ -55,38 +55,11 @@ export class InMemoryIndexer implements StorageTypes.IIndexer {
};
}

async getTransactionsByTopics(
topics: string[],
page?: number,
pageSize?: number,
): Promise<StorageTypes.IGetTransactionsResponse> {
if (page !== undefined && page < 1) {
throw new Error('Page must be greater than or equal to 1');
}
if (pageSize !== undefined && pageSize <= 0) {
throw new Error('Page size must be greater than 0');
}

async getTransactionsByTopics(topics: string[]): Promise<StorageTypes.IGetTransactionsResponse> {
// Efficiently get total count without creating intermediate array
const channelIdsSet = new Set(topics.flatMap((topic) => this.#topicToChannelsIndex.get(topic)));
const total = channelIdsSet.size;
let channelIds = Array.from(channelIdsSet);
const channelIds = Array.from(channelIdsSet);

if (page && pageSize) {
const start = (page - 1) * pageSize;
// Return empty result if page exceeds available data
if (start >= total) {
return {
blockNumber: 0,
transactions: [],
pagination:
page && pageSize
? { total, page, pageSize, hasMore: page * pageSize < total }
: undefined,
};
}
channelIds = channelIds.slice(start, start + pageSize);
}
const locations = channelIds
.map((channel) => this.#channelToLocationsIndex.get(channel))
.flat();
9 changes: 6 additions & 3 deletions packages/integration-test/test/node-client.test.ts
Original file line number Diff line number Diff line change
@@ -273,7 +273,8 @@ describe('Request client using a request node', () => {
await waitForConfirmation(requestDataCancel);

// get requests without boundaries
let requests = await requestNetwork.fromTopic(topicsRequest1and2[0]);
const response = await requestNetwork.fromTopic(topicsRequest1and2[0]);
let requests = Array.isArray(response) ? response : response.requests;
expect(requests.length).toBe(2);
expect(requests[0].requestId).toBe(request1.requestId);
expect(requests[1].requestId).toBe(request2.requestId);
@@ -286,9 +287,10 @@ describe('Request client using a request node', () => {
expect(requestData2.state).toBe(Types.RequestLogic.STATE.CREATED);

// get requests with boundaries
requests = await requestNetwork.fromTopic(topicsRequest1and2[0], {
const result = await requestNetwork.fromTopic(topicsRequest1and2[0], {
to: timestampBeforeReduce,
});
requests = Array.isArray(result) ? result : result.requests;
expect(requests.length).toBe(1);
expect(requests[0].requestId).toBe(request1.requestId);

@@ -341,9 +343,10 @@ describe('Request client using a request node', () => {
await new Promise((r) => setTimeout(r, 1500));

// get requests with boundaries
const requests = await requestNetwork.fromIdentity(payerSmartContract, {
const result = await requestNetwork.fromIdentity(payerSmartContract, {
from: timestampCreation,
});
const requests = Array.isArray(result) ? result : result.requests;
expect(requests.length).toBe(1);
expect(requests[0].requestId).toBe(request1.requestId);
});
45 changes: 32 additions & 13 deletions packages/request-client.js/src/api/request-network.ts
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ import {
SignatureProviderTypes,
TransactionTypes,
} from '@requestnetwork/types';
import { deepCopy, supportedIdentities, validatePaginationParams } from '@requestnetwork/utils';
import { deepCopy, supportedIdentities } from '@requestnetwork/utils';
import { CurrencyManager, UnsupportedCurrencyError } from '@requestnetwork/currency';
import * as Types from '../types';
import ContentDataExtension from './content-data-extension';
@@ -294,7 +294,9 @@ export default class RequestNetwork {
page?: number;
pageSize?: number;
},
): Promise<Request[]> {
): Promise<
Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
> {
if (!this.supportedIdentities.includes(identity.type)) {
throw new Error(`${identity.type} is not supported`);
}
@@ -317,7 +319,9 @@ export default class RequestNetwork {
page?: number;
pageSize?: number;
},
): Promise<Request[]> {
): Promise<
Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
> {
const identityNotSupported = identities.find(
(identity) => !this.supportedIdentities.includes(identity.type),
);
@@ -345,9 +349,9 @@ export default class RequestNetwork {
page?: number;
pageSize?: number;
},
): Promise<Request[]> {
validatePaginationParams(options?.page, options?.pageSize);

): Promise<
Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
> {
// Gets all the requests indexed by the value of the identity
const requestsAndMeta: RequestLogicTypes.IReturnGetRequestsByTopic =
await this.requestLogic.getRequestsByTopic(
@@ -389,8 +393,16 @@ export default class RequestNetwork {
return request;
},
);

return Promise.all(requestPromises);
const requests = await Promise.all(requestPromises);

if (options?.page && options?.pageSize) {
return {
requests,
meta: requestsAndMeta.meta,
};
} else {
return requests;
}
}

/**
@@ -409,9 +421,9 @@ export default class RequestNetwork {
page?: number;
pageSize?: number;
},
): Promise<Request[]> {
validatePaginationParams(options?.page, options?.pageSize);

): Promise<
Request[] | { meta: RequestLogicTypes.IReturnGetRequestsByTopic['meta']; requests: Request[] }
> {
// Gets all the requests indexed by the value of the identity
const requestsAndMeta: RequestLogicTypes.IReturnGetRequestsByTopic =
await this.requestLogic.getRequestsByMultipleTopics(
@@ -454,8 +466,15 @@ export default class RequestNetwork {
return request;
},
);

return Promise.all(requestPromises);
const requests = await Promise.all(requestPromises);
if (options?.page && options?.pageSize) {
return {
requests,
meta: requestsAndMeta.meta,
};
} else {
return requests;
}
}

/*
8 changes: 2 additions & 6 deletions packages/request-client.js/src/http-data-access.ts
Original file line number Diff line number Diff line change
@@ -110,10 +110,8 @@ export default class HttpDataAccess extends CombinedDataAccess {
public async getChannelsByTopic(
topic: string,
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
return await this.reader.getChannelsByTopic(topic, updatedBetween, page, pageSize);
return await this.reader.getChannelsByTopic(topic, updatedBetween);
}

/**
@@ -125,10 +123,8 @@ export default class HttpDataAccess extends CombinedDataAccess {
public async getChannelsByMultipleTopics(
topics: string[],
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween, page, pageSize);
return await this.reader.getChannelsByMultipleTopics(topics, updatedBetween);
}

/**
13 changes: 0 additions & 13 deletions packages/request-client.js/src/http-data-read.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import { DataAccessTypes } from '@requestnetwork/types';
import { validatePaginationParams } from '@requestnetwork/utils';
import { HttpDataAccessConfig } from './http-data-access-config';

export class HttpDataRead implements DataAccessTypes.IDataRead {
@@ -53,16 +52,10 @@ export class HttpDataRead implements DataAccessTypes.IDataRead {
public async getChannelsByTopic(
topic: string,
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
validatePaginationParams(page, pageSize);

const params = {
topic,
updatedBetween,
...(page !== undefined && { page }),
...(pageSize !== undefined && { pageSize }),
};

return await this.dataAccessConfig.fetchAndRetry('/getChannelsByTopic', params);
@@ -77,16 +70,10 @@ export class HttpDataRead implements DataAccessTypes.IDataRead {
public async getChannelsByMultipleTopics(
topics: string[],
updatedBetween?: DataAccessTypes.ITimestampBoundaries,
page?: number,
pageSize?: number,
): Promise<DataAccessTypes.IReturnGetChannelsByTopic> {
validatePaginationParams(page, pageSize);

return await this.dataAccessConfig.fetchAndRetry('/getChannelsByMultipleTopics', {
topics,
updatedBetween,
page,
pageSize,
});
}
}
Loading