Skip to content

Commit 88fa9a7

Browse files
sfc-gh-yzousfc-gh-yzou
authored
revert pagenation (apache#46)
1 parent 3bf8d69 commit 88fa9a7

File tree

32 files changed

+197
-826
lines changed

32 files changed

+197
-826
lines changed

‎extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java‎

Lines changed: 18 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@
3737
import java.util.function.Predicate;
3838
import java.util.function.Supplier;
3939
import java.util.stream.Collectors;
40-
import java.util.stream.Stream;
4140
import org.apache.polaris.core.PolarisCallContext;
4241
import org.apache.polaris.core.context.RealmContext;
4342
import org.apache.polaris.core.entity.EntityNameLookupRecord;
@@ -53,9 +52,6 @@
5352
import org.apache.polaris.core.persistence.BaseMetaStoreManager;
5453
import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
5554
import org.apache.polaris.core.persistence.RetryOnConcurrencyException;
56-
import org.apache.polaris.core.persistence.pagination.HasPageSize;
57-
import org.apache.polaris.core.persistence.pagination.Page;
58-
import org.apache.polaris.core.persistence.pagination.PageToken;
5955
import org.apache.polaris.core.persistence.transactional.AbstractTransactionalPersistence;
6056
import org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
6157
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
@@ -423,30 +419,29 @@ public List<EntityNameLookupRecord> lookupEntityActiveBatchInCurrentTxn(
423419

424420
/** {@inheritDoc} */
425421
@Override
426-
public @Nonnull Page<EntityNameLookupRecord> listEntitiesInCurrentTxn(
422+
public @Nonnull List<EntityNameLookupRecord> listEntitiesInCurrentTxn(
427423
@Nonnull PolarisCallContext callCtx,
428424
long catalogId,
429425
long parentId,
430-
@Nonnull PolarisEntityType entityType,
431-
@Nonnull PageToken pageToken) {
426+
@Nonnull PolarisEntityType entityType) {
432427
return this.listEntitiesInCurrentTxn(
433-
callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue(), pageToken);
428+
callCtx, catalogId, parentId, entityType, Predicates.alwaysTrue());
434429
}
435430

436431
@Override
437-
public @Nonnull Page<EntityNameLookupRecord> listEntitiesInCurrentTxn(
432+
public @Nonnull List<EntityNameLookupRecord> listEntitiesInCurrentTxn(
438433
@Nonnull PolarisCallContext callCtx,
439434
long catalogId,
440435
long parentId,
441436
@Nonnull PolarisEntityType entityType,
442-
@Nonnull Predicate<PolarisBaseEntity> entityFilter,
443-
@Nonnull PageToken pageToken) {
437+
@Nonnull Predicate<PolarisBaseEntity> entityFilter) {
444438
// full range scan under the parent for that type
445439
return this.listEntitiesInCurrentTxn(
446440
callCtx,
447441
catalogId,
448442
parentId,
449443
entityType,
444+
Integer.MAX_VALUE,
450445
entityFilter,
451446
entity ->
452447
new EntityNameLookupRecord(
@@ -455,33 +450,27 @@ public List<EntityNameLookupRecord> lookupEntityActiveBatchInCurrentTxn(
455450
entity.getParentId(),
456451
entity.getName(),
457452
entity.getTypeCode(),
458-
entity.getSubTypeCode()),
459-
pageToken);
453+
entity.getSubTypeCode()));
460454
}
461455

462456
@Override
463-
public @Nonnull <T> Page<T> listEntitiesInCurrentTxn(
457+
public @Nonnull <T> List<T> listEntitiesInCurrentTxn(
464458
@Nonnull PolarisCallContext callCtx,
465459
long catalogId,
466460
long parentId,
467461
@Nonnull PolarisEntityType entityType,
462+
int limit,
468463
@Nonnull Predicate<PolarisBaseEntity> entityFilter,
469-
@Nonnull Function<PolarisBaseEntity, T> transformer,
470-
@Nonnull PageToken pageToken) {
464+
@Nonnull Function<PolarisBaseEntity, T> transformer) {
471465
// full range scan under the parent for that type
472-
Stream<PolarisBaseEntity> data =
473-
this.store
474-
.lookupFullEntitiesActive(
475-
localSession.get(), catalogId, parentId, entityType, pageToken)
476-
.stream()
477-
.map(ModelEntity::toEntity)
478-
.filter(entityFilter);
479-
480-
if (pageToken instanceof HasPageSize hasPageSize) {
481-
data = data.limit(hasPageSize.getPageSize());
482-
}
483-
484-
return Page.fromItems(data.map(transformer).collect(Collectors.toList()));
466+
return this.store
467+
.lookupFullEntitiesActive(localSession.get(), catalogId, parentId, entityType)
468+
.stream()
469+
.map(ModelEntity::toEntity)
470+
.filter(entityFilter)
471+
.limit(limit)
472+
.map(transformer)
473+
.collect(Collectors.toList());
485474
}
486475

487476
/** {@inheritDoc} */

‎extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkStore.java‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535
import org.apache.polaris.core.entity.PolarisEntityType;
3636
import org.apache.polaris.core.entity.PolarisGrantRecord;
3737
import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
38-
import org.apache.polaris.core.persistence.pagination.PageToken;
3938
import org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
4039
import org.apache.polaris.jpa.models.ModelEntity;
4140
import org.apache.polaris.jpa.models.ModelEntityActive;
@@ -283,11 +282,7 @@ long countActiveChildEntities(
283282
}
284283

285284
List<ModelEntity> lookupFullEntitiesActive(
286-
EntityManager session,
287-
long catalogId,
288-
long parentId,
289-
@Nonnull PolarisEntityType entityType,
290-
@Nonnull PageToken pageToken) {
285+
EntityManager session, long catalogId, long parentId, @Nonnull PolarisEntityType entityType) {
291286
diagnosticServices.check(session != null, "session_is_null");
292287
checkInitialized();
293288

‎extension/persistence/relational-jdbc/src/main/java/org/apache/polaris/extension/persistence/relational/jdbc/JdbcBasePersistenceImpl.java‎

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,6 @@
4949
import org.apache.polaris.core.persistence.PolicyMappingAlreadyExistsException;
5050
import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
5151
import org.apache.polaris.core.persistence.RetryOnConcurrencyException;
52-
import org.apache.polaris.core.persistence.pagination.HasPageSize;
53-
import org.apache.polaris.core.persistence.pagination.Page;
54-
import org.apache.polaris.core.persistence.pagination.PageToken;
5552
import org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
5653
import org.apache.polaris.core.policy.PolicyType;
5754
import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
@@ -354,51 +351,49 @@ public List<PolarisChangeTrackingVersions> lookupEntityVersions(
354351

355352
@Nonnull
356353
@Override
357-
public Page<EntityNameLookupRecord> listEntities(
354+
public List<EntityNameLookupRecord> listEntities(
358355
@Nonnull PolarisCallContext callCtx,
359356
long catalogId,
360357
long parentId,
361-
@Nonnull PolarisEntityType entityType,
362-
@Nonnull PageToken pageToken) {
358+
@Nonnull PolarisEntityType entityType) {
363359
return listEntities(
364360
callCtx,
365361
catalogId,
366362
parentId,
367363
entityType,
364+
Integer.MAX_VALUE,
368365
entity -> true,
369-
EntityNameLookupRecord::new,
370-
pageToken);
366+
EntityNameLookupRecord::new);
371367
}
372368

373369
@Nonnull
374370
@Override
375-
public Page<EntityNameLookupRecord> listEntities(
371+
public List<EntityNameLookupRecord> listEntities(
376372
@Nonnull PolarisCallContext callCtx,
377373
long catalogId,
378374
long parentId,
379375
@Nonnull PolarisEntityType entityType,
380-
@Nonnull Predicate<PolarisBaseEntity> entityFilter,
381-
@Nonnull PageToken pageToken) {
376+
@Nonnull Predicate<PolarisBaseEntity> entityFilter) {
382377
return listEntities(
383378
callCtx,
384379
catalogId,
385380
parentId,
386381
entityType,
382+
Integer.MAX_VALUE,
387383
entityFilter,
388-
EntityNameLookupRecord::new,
389-
pageToken);
384+
EntityNameLookupRecord::new);
390385
}
391386

392387
@Nonnull
393388
@Override
394-
public <T> Page<T> listEntities(
389+
public <T> List<T> listEntities(
395390
@Nonnull PolarisCallContext callCtx,
396391
long catalogId,
397392
long parentId,
398393
PolarisEntityType entityType,
394+
int limit,
399395
@Nonnull Predicate<PolarisBaseEntity> entityFilter,
400-
@Nonnull Function<PolarisBaseEntity, T> transformer,
401-
@Nonnull PageToken pageToken) {
396+
@Nonnull Function<PolarisBaseEntity, T> transformer) {
402397
Map<String, Object> params =
403398
Map.of(
404399
"catalog_id",
@@ -419,17 +414,11 @@ public <T> Page<T> listEntities(
419414
query,
420415
new ModelEntity(),
421416
stream -> {
422-
var data = stream.filter(entityFilter);
423-
if (pageToken instanceof HasPageSize hasPageSize) {
424-
data = data.limit(hasPageSize.getPageSize());
425-
}
426-
data.forEach(results::add);
417+
stream.filter(entityFilter).limit(limit).forEach(results::add);
427418
});
428-
List<T> resultsOrEmpty =
429-
results == null
430-
? Collections.emptyList()
431-
: results.stream().filter(entityFilter).map(transformer).collect(Collectors.toList());
432-
return Page.fromItems(resultsOrEmpty);
419+
return results == null
420+
? Collections.emptyList()
421+
: results.stream().filter(entityFilter).map(transformer).collect(Collectors.toList());
433422
} catch (SQLException e) {
434423
throw new RuntimeException(
435424
String.format("Failed to retrieve polaris entities due to %s", e.getMessage()), e);

‎polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -214,13 +214,6 @@ public static void enforceFeatureEnabledOrThrow(
214214
.defaultValue(2)
215215
.buildFeatureConfiguration();
216216

217-
public static final PolarisConfiguration<Boolean> LIST_PAGINATION_ENABLED =
218-
PolarisConfiguration.<Boolean>builder()
219-
.key("LIST_PAGINATION_ENABLED")
220-
.description("If set to true, pagination for APIs like listTables is enabled.")
221-
.defaultValue(false)
222-
.buildFeatureConfiguration();
223-
224217
public static final FeatureConfiguration<Boolean> ENABLE_GENERIC_TABLES =
225218
PolarisConfiguration.<Boolean>builder()
226219
.key("ENABLE_GENERIC_TABLES")

‎polaris-core/src/main/java/org/apache/polaris/core/persistence/AtomicOperationMetaStoreManager.java‎

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,6 @@
6262
import org.apache.polaris.core.persistence.dao.entity.PrivilegeResult;
6363
import org.apache.polaris.core.persistence.dao.entity.ResolvedEntityResult;
6464
import org.apache.polaris.core.persistence.dao.entity.ScopedCredentialsResult;
65-
import org.apache.polaris.core.persistence.pagination.Page;
66-
import org.apache.polaris.core.persistence.pagination.PageToken;
6765
import org.apache.polaris.core.policy.PolarisPolicyMappingRecord;
6866
import org.apache.polaris.core.policy.PolicyEntity;
6967
import org.apache.polaris.core.policy.PolicyMappingUtil;
@@ -689,8 +687,7 @@ private void revokeGrantRecord(
689687
@Nonnull PolarisCallContext callCtx,
690688
@Nullable List<PolarisEntityCore> catalogPath,
691689
@Nonnull PolarisEntityType entityType,
692-
@Nonnull PolarisEntitySubType entitySubType,
693-
@Nonnull PageToken pageToken) {
690+
@Nonnull PolarisEntitySubType entitySubType) {
694691
// get meta store we should be using
695692
BasePersistence ms = callCtx.getMetaStore();
696693

@@ -702,16 +699,15 @@ private void revokeGrantRecord(
702699
catalogPath == null || catalogPath.size() == 0
703700
? 0l
704701
: catalogPath.get(catalogPath.size() - 1).getId();
705-
Page<EntityNameLookupRecord> resultPage =
706-
ms.listEntities(callCtx, catalogId, parentId, entityType, pageToken);
702+
List<EntityNameLookupRecord> toreturnList =
703+
ms.listEntities(callCtx, catalogId, parentId, entityType);
707704

708705
// prune the returned list with only entities matching the entity subtype
709706
if (entitySubType != PolarisEntitySubType.ANY_SUBTYPE) {
710-
resultPage =
711-
pageToken.buildNextPage(
712-
resultPage.items.stream()
713-
.filter(rec -> rec.getSubTypeCode() == entitySubType.getCode())
714-
.collect(Collectors.toList()));
707+
toreturnList =
708+
toreturnList.stream()
709+
.filter(rec -> rec.getSubTypeCode() == entitySubType.getCode())
710+
.collect(Collectors.toList());
715711
}
716712

717713
// TODO: Use post-validation to enforce consistent view against catalogPath. In the
@@ -721,7 +717,7 @@ private void revokeGrantRecord(
721717
// in-flight request (the cache-based resolution follows a different path entirely).
722718

723719
// done
724-
return ListEntitiesResult.fromPage(resultPage);
720+
return new ListEntitiesResult(toreturnList);
725721
}
726722

727723
/** {@inheritDoc} */
@@ -1180,14 +1176,13 @@ private void revokeGrantRecord(
11801176
// get the list of catalog roles, at most 2
11811177
List<PolarisBaseEntity> catalogRoles =
11821178
ms.listEntities(
1183-
callCtx,
1184-
catalogId,
1185-
catalogId,
1186-
PolarisEntityType.CATALOG_ROLE,
1187-
entity -> true,
1188-
Function.identity(),
1189-
PageToken.fromLimit(2))
1190-
.items;
1179+
callCtx,
1180+
catalogId,
1181+
catalogId,
1182+
PolarisEntityType.CATALOG_ROLE,
1183+
2,
1184+
entity -> true,
1185+
Function.identity());
11911186

11921187
// if we have 2, we cannot drop the catalog. If only one left, better be the admin role
11931188
if (catalogRoles.size() > 1) {
@@ -1493,16 +1488,17 @@ private void revokeGrantRecord(
14931488

14941489
@Override
14951490
public @Nonnull EntitiesResult loadTasks(
1496-
@Nonnull PolarisCallContext callCtx, String executorId, PageToken pageToken) {
1491+
@Nonnull PolarisCallContext callCtx, String executorId, int limit) {
14971492
BasePersistence ms = callCtx.getMetaStore();
14981493

14991494
// find all available tasks
1500-
Page<PolarisBaseEntity> availableTasks =
1495+
List<PolarisBaseEntity> availableTasks =
15011496
ms.listEntities(
15021497
callCtx,
15031498
PolarisEntityConstants.getRootEntityId(),
15041499
PolarisEntityConstants.getRootEntityId(),
15051500
PolarisEntityType.TASK,
1501+
limit,
15061502
entity -> {
15071503
PolarisObjectMapperUtil.TaskExecutionState taskState =
15081504
PolarisObjectMapperUtil.parseTaskState(entity);
@@ -1517,12 +1513,11 @@ private void revokeGrantRecord(
15171513
|| taskState.executor == null
15181514
|| callCtx.getClock().millis() - taskState.lastAttemptStartTime > taskAgeTimeout;
15191515
},
1520-
Function.identity(),
1521-
pageToken);
1516+
Function.identity());
15221517

15231518
List<PolarisBaseEntity> loadedTasks = new ArrayList<>();
15241519
final AtomicInteger failedLeaseCount = new AtomicInteger(0);
1525-
availableTasks.items.forEach(
1520+
availableTasks.forEach(
15261521
task -> {
15271522
PolarisBaseEntity updatedTask = new PolarisBaseEntity(task);
15281523
Map<String, String> properties =
@@ -1559,7 +1554,7 @@ private void revokeGrantRecord(
15591554
throw new RetryOnConcurrencyException(
15601555
"Failed to lease any of %s tasks due to concurrent leases", failedLeaseCount.get());
15611556
}
1562-
return EntitiesResult.fromPage(Page.fromItems(loadedTasks));
1557+
return new EntitiesResult(loadedTasks);
15631558
}
15641559

15651560
/** {@inheritDoc} */

0 commit comments

Comments
 (0)