diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 80203ef59095c..c819ae73e0ae8 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -55,6 +55,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.function.BiPredicate; import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.Collectors; @@ -113,7 +114,12 @@ protected void masterOperation( getMultipleReposSnapshotInfo( request.isSingleRepositoryRequest() == false, state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY), - TransportGetRepositoriesAction.getRepositories(state, request.repositories()), + maybeFilterRepositories( + TransportGetRepositoriesAction.getRepositories(state, request.repositories()), + request.sort(), + request.order(), + request.fromSortValue() + ), request.snapshots(), request.ignoreUnavailable(), request.verbose(), @@ -123,11 +129,31 @@ protected void masterOperation( request.offset(), request.size(), request.order(), - buildSnapshotPredicate(request.sort(), request.order(), request.policies(), request.fromSortValue()), + new SnapshotPredicates(request), listener ); } + /** + * Filters the list of repositories that a request will fetch snapshots from in the special case of sorting by repository + * name and having a non-null value for {@link GetSnapshotsRequest#fromSortValue()} on the request to exclude repositories outside + * the sort value range if possible. + */ + private static List maybeFilterRepositories( + List repositories, + GetSnapshotsRequest.SortBy sortBy, + SortOrder order, + @Nullable String fromSortValue + ) { + if (sortBy != GetSnapshotsRequest.SortBy.REPOSITORY || fromSortValue == null) { + return repositories; + } + final Predicate predicate = order == SortOrder.ASC + ? repositoryMetadata -> fromSortValue.compareTo(repositoryMetadata.name()) <= 0 + : repositoryMetadata -> fromSortValue.compareTo(repositoryMetadata.name()) >= 0; + return Collections.unmodifiableList(repositories.stream().filter(predicate).collect(Collectors.toList())); + } + private void getMultipleReposSnapshotInfo( boolean isMultiRepoRequest, SnapshotsInProgress snapshotsInProgress, @@ -141,7 +167,7 @@ private void getMultipleReposSnapshotInfo( int offset, int size, SortOrder order, - @Nullable Predicate predicate, + SnapshotPredicates predicates, ActionListener listener ) { // short-circuit if there are no repos, because we can not create GroupedActionListener of size 0 @@ -161,7 +187,7 @@ private void getMultipleReposSnapshotInfo( .map(Tuple::v1) .filter(Objects::nonNull) .collect(Collectors.toMap(Tuple::v1, Tuple::v2)); - final SnapshotsInRepo snInfos = sortAndFilterSnapshots(allSnapshots, sortBy, after, offset, size, order, predicate); + final SnapshotsInRepo snInfos = sortSnapshots(allSnapshots, sortBy, after, offset, size, order); final List snapshotInfos = snInfos.snapshotInfos; final int remaining = snInfos.remaining + responses.stream() .map(Tuple::v2) @@ -185,7 +211,7 @@ private void getMultipleReposSnapshotInfo( snapshotsInProgress, repoName, snapshots, - predicate, + predicates, ignoreUnavailable, verbose, cancellableTask, @@ -207,7 +233,7 @@ private void getSingleRepoSnapshotInfo( SnapshotsInProgress snapshotsInProgress, String repo, String[] snapshots, - Predicate predicate, + SnapshotPredicates predicates, boolean ignoreUnavailable, boolean verbose, CancellableTask task, @@ -245,7 +271,7 @@ private void getSingleRepoSnapshotInfo( sortBy, after, order, - predicate, + predicates, listener ), listener::onFailure @@ -285,16 +311,19 @@ private void loadSnapshotInfos( GetSnapshotsRequest.SortBy sortBy, @Nullable final GetSnapshotsRequest.After after, SortOrder order, - @Nullable Predicate predicate, + SnapshotPredicates predicates, ActionListener listener ) { if (task.notifyIfCancelled(listener)) { return; } + final BiPredicate preflightPredicate = predicates.preflightPredicate(); if (repositoryData != null) { for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) { - allSnapshotIds.put(snapshotId.getName(), new Snapshot(repo, snapshotId)); + if (preflightPredicate == null || preflightPredicate.test(snapshotId, repositoryData)) { + allSnapshotIds.put(snapshotId.getName(), new Snapshot(repo, snapshotId)); + } } } @@ -357,11 +386,11 @@ private void loadSnapshotInfos( sortBy, after, order, - predicate, + predicates.snapshotPredicate(), listener ); } else { - assert predicate == null : "filtering is not supported in non-verbose mode"; + assert predicates.snapshotPredicate() == null : "filtering is not supported in non-verbose mode"; final SnapshotsInRepo snapshotInfos; if (repositoryData != null) { // want non-current snapshots as well, which are found in the repository data @@ -413,7 +442,10 @@ private void snapshots( ); for (SnapshotsInProgress.Entry entry : entries) { if (snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId())) { - snapshotSet.add(new SnapshotInfo(entry)); + final SnapshotInfo snapshotInfo = new SnapshotInfo(entry); + if (predicate == null || predicate.test(snapshotInfo)) { + snapshotSet.add(new SnapshotInfo(entry)); + } } } // then, look in the repository if there's any matching snapshots left @@ -426,7 +458,7 @@ private void snapshots( final ActionListener allDoneListener = listener.delegateFailure((l, v) -> { final ArrayList snapshotList = new ArrayList<>(snapshotInfos); snapshotList.addAll(snapshotSet); - listener.onResponse(sortAndFilterSnapshots(snapshotList, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order, predicate)); + listener.onResponse(sortSnapshots(snapshotList, sortBy, after, 0, GetSnapshotsRequest.NO_LIMIT, order)); }); if (snapshotIdsToIterate.isEmpty()) { allDoneListener.onResponse(null); @@ -444,7 +476,11 @@ private void snapshots( snapshotIdsToIterate, ignoreUnavailable == false, task::isCancelled, - (context, snapshotInfo) -> snapshotInfos.add(snapshotInfo), + predicate == null ? (context, snapshotInfo) -> snapshotInfos.add(snapshotInfo) : (context, snapshotInfo) -> { + if (predicate.test(snapshotInfo)) { + snapshotInfos.add(snapshotInfo); + } + }, allDoneListener ) ); @@ -514,43 +550,36 @@ private static SnapshotsInRepo buildSimpleSnapshotInfos( private static final Comparator BY_REPOSITORY = Comparator.comparing(SnapshotInfo::repository) .thenComparing(SnapshotInfo::snapshotId); - private static SnapshotsInRepo sortAndFilterSnapshots( - final List snapshotInfos, - final GetSnapshotsRequest.SortBy sortBy, - final @Nullable GetSnapshotsRequest.After after, - final int offset, - final int size, - final SortOrder order, - final @Nullable Predicate predicate - ) { - final List filteredSnapshotInfos; - if (predicate == null) { - filteredSnapshotInfos = snapshotInfos; - } else { - filteredSnapshotInfos = Collections.unmodifiableList(snapshotInfos.stream().filter(predicate).collect(Collectors.toList())); + private static long getDuration(SnapshotId snapshotId, RepositoryData repositoryData) { + final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); + if (details == null) { + return -1; + } + final long startTime = details.getStartTimeMillis(); + if (startTime == -1) { + return -1; } - return sortSnapshots(filteredSnapshotInfos, sortBy, after, offset, size, order); + final long endTime = details.getEndTimeMillis(); + if (endTime == -1) { + return -1; + } + return endTime - startTime; } - private static Predicate buildSnapshotPredicate( - GetSnapshotsRequest.SortBy sortBy, - SortOrder order, - String[] slmPolicies, - String fromSortValue - ) { - Predicate predicate = null; - if (slmPolicies.length > 0) { - predicate = filterBySLMPolicies(slmPolicies); - } - if (fromSortValue != null) { - final Predicate fromSortValuePredicate = buildFromSortValuePredicate(sortBy, fromSortValue, order, null, null); - if (predicate == null) { - predicate = fromSortValuePredicate; - } else { - predicate = fromSortValuePredicate.and(predicate); + private static long getStartTime(SnapshotId snapshotId, RepositoryData repositoryData) { + final RepositoryData.SnapshotDetails details = repositoryData.getSnapshotDetails(snapshotId); + return details == null ? -1 : details.getStartTimeMillis(); + } + + private static int indexCount(SnapshotId snapshotId, RepositoryData repositoryData) { + // TODO: this could be made more efficient by caching this number in RepositoryData + int indexCount = 0; + for (IndexId idx : repositoryData.getIndices().values()) { + if (repositoryData.getSnapshots(idx).contains(snapshotId)) { + indexCount++; } } - return predicate; + return indexCount; } private static SnapshotsInRepo sortSnapshots( @@ -592,7 +621,7 @@ private static SnapshotsInRepo sortSnapshots( if (after != null) { assert offset == 0 : "can't combine after and offset but saw [" + after + "] and offset [" + offset + "]"; - infos = infos.filter(buildFromSortValuePredicate(sortBy, after.value(), order, after.snapshotName(), after.repoName())); + infos = infos.filter(buildAfterPredicate(sortBy, after, order)); } infos = infos.sorted(order == SortOrder.DESC ? comparator.reversed() : comparator).skip(offset); final List allSnapshots = infos.collect(Collectors.toList()); @@ -608,64 +637,39 @@ private static SnapshotsInRepo sortSnapshots( return new SnapshotsInRepo(resultSet, snapshotInfos.size(), allSnapshots.size() - resultSet.size()); } - private static Predicate buildFromSortValuePredicate( + private static Predicate buildAfterPredicate( GetSnapshotsRequest.SortBy sortBy, - String after, - SortOrder order, - @Nullable String snapshotName, - @Nullable String repoName + GetSnapshotsRequest.After after, + SortOrder order ) { - final Predicate isAfter; + final String snapshotName = after.snapshotName(); + final String repoName = after.repoName(); + final String value = after.value(); switch (sortBy) { case START_TIME: - isAfter = filterByLongOffset(SnapshotInfo::startTime, Long.parseLong(after), snapshotName, repoName, order); - break; + return filterByLongOffset(SnapshotInfo::startTime, Long.parseLong(value), snapshotName, repoName, order); case NAME: - if (snapshotName == null) { - assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; - isAfter = order == SortOrder.ASC - ? snapshotInfo -> after.compareTo(snapshotInfo.snapshotId().getName()) <= 0 - : snapshotInfo -> after.compareTo(snapshotInfo.snapshotId().getName()) >= 0; - } else { - isAfter = order == SortOrder.ASC - ? (info -> compareName(snapshotName, repoName, info) < 0) - : (info -> compareName(snapshotName, repoName, info) > 0); - } - break; + // TODO: cover via pre-flight predicate + return order == SortOrder.ASC + ? (info -> compareName(snapshotName, repoName, info) < 0) + : (info -> compareName(snapshotName, repoName, info) > 0); case DURATION: - isAfter = filterByLongOffset( - info -> info.endTime() - info.startTime(), - Long.parseLong(after), - snapshotName, - repoName, - order - ); - break; + return filterByLongOffset(info -> info.endTime() - info.startTime(), Long.parseLong(value), snapshotName, repoName, order); case INDICES: - isAfter = filterByLongOffset(info -> info.indices().size(), Integer.parseInt(after), snapshotName, repoName, order); - break; + // TODO: cover via pre-flight predicate + return filterByLongOffset(info -> info.indices().size(), Integer.parseInt(value), snapshotName, repoName, order); case SHARDS: - isAfter = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(after), snapshotName, repoName, order); - break; + return filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(value), snapshotName, repoName, order); case FAILED_SHARDS: - isAfter = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(after), snapshotName, repoName, order); - break; + return filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(value), snapshotName, repoName, order); case REPOSITORY: - if (snapshotName == null) { - assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; - isAfter = order == SortOrder.ASC - ? snapshotInfo -> after.compareTo(snapshotInfo.repository()) <= 0 - : snapshotInfo -> after.compareTo(snapshotInfo.repository()) >= 0; - } else { - isAfter = order == SortOrder.ASC - ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) - : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); - } - break; + // TODO: cover via pre-flight predicate + return order == SortOrder.ASC + ? (info -> compareRepositoryName(snapshotName, repoName, info) < 0) + : (info -> compareRepositoryName(snapshotName, repoName, info) > 0); default: throw new AssertionError("unexpected sort column [" + sortBy + "]"); } - return isAfter; } private static Predicate filterBySLMPolicies(String[] slmPolicies) { @@ -707,20 +711,19 @@ private static Predicate filterBySLMPolicies(String[] slmPolicies) }; } + private static Predicate filterByLongOffset(ToLongFunction extractor, long after, SortOrder order) { + return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info); + } + private static Predicate filterByLongOffset( ToLongFunction extractor, long after, - @Nullable String snapshotName, - @Nullable String repoName, + String snapshotName, + String repoName, SortOrder order ) { - if (snapshotName == null) { - assert repoName == null : "no snapshot name given but saw repo name [" + repoName + "]"; - return order == SortOrder.ASC ? info -> after <= extractor.applyAsLong(info) : info -> after >= extractor.applyAsLong(info); - } return order == SortOrder.ASC ? info -> { final long val = extractor.applyAsLong(info); - return after < val || (after == val && compareName(snapshotName, repoName, info) < 0); } : info -> { final long val = extractor.applyAsLong(info); @@ -744,6 +747,107 @@ private static int compareName(String name, String repoName, SnapshotInfo info) return repoName.compareTo(info.repository()); } + /** + * A pair of predicates for the get snapshots action. The {@link #preflightPredicate()} is applied to combinations of snapshot id and + * repository data to determine which snapshots to fully load from the repository and rules out all snapshots that do not match the + * given {@link GetSnapshotsRequest} that can be ruled out through the information in {@link RepositoryData}. + * The predicate returned by {@link #snapshotPredicate()} is then applied the instances of {@link SnapshotInfo} that were loaded from + * the repository to filter out those remaining that did not match the request but could not be ruled out without loading their + * {@link SnapshotInfo}. + */ + private static final class SnapshotPredicates { + + private final Predicate snapshotPredicate; + + private final BiPredicate preflightPredicate; + + SnapshotPredicates(GetSnapshotsRequest request) { + Predicate snapshotPredicate = null; + final String[] slmPolicies = request.policies(); + final String fromSortValue = request.fromSortValue(); + if (slmPolicies.length > 0) { + snapshotPredicate = filterBySLMPolicies(slmPolicies); + } + final GetSnapshotsRequest.SortBy sortBy = request.sort(); + final SortOrder order = request.order(); + if (fromSortValue == null) { + preflightPredicate = null; + } else { + final Predicate fromSortValuePredicate; + switch (sortBy) { + case START_TIME: + final long after = Long.parseLong(fromSortValue); + preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> { + final long startTime = getStartTime(snapshotId, repositoryData); + return startTime == -1 || after <= startTime; + } : (snapshotId, repositoryData) -> { + final long startTime = getStartTime(snapshotId, repositoryData); + return startTime == -1 || after >= startTime; + }; + fromSortValuePredicate = filterByLongOffset(SnapshotInfo::startTime, after, order); + break; + case NAME: + preflightPredicate = order == SortOrder.ASC + ? (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) <= 0 + : (snapshotId, repositoryData) -> fromSortValue.compareTo(snapshotId.getName()) >= 0; + fromSortValuePredicate = null; + break; + case DURATION: + final long afterDuration = Long.parseLong(fromSortValue); + preflightPredicate = order == SortOrder.ASC ? (snapshotId, repositoryData) -> { + final long duration = getDuration(snapshotId, repositoryData); + return duration == -1 || afterDuration <= duration; + } : (snapshotId, repositoryData) -> { + final long duration = getDuration(snapshotId, repositoryData); + return duration == -1 || afterDuration >= duration; + }; + fromSortValuePredicate = filterByLongOffset(info -> info.endTime() - info.startTime(), afterDuration, order); + break; + case INDICES: + final int afterIndexCount = Integer.parseInt(fromSortValue); + preflightPredicate = order == SortOrder.ASC + ? (snapshotId, repositoryData) -> afterIndexCount <= indexCount(snapshotId, repositoryData) + : (snapshotId, repositoryData) -> afterIndexCount >= indexCount(snapshotId, repositoryData); + fromSortValuePredicate = null; + break; + case REPOSITORY: + // already handled in #maybeFilterRepositories + preflightPredicate = null; + fromSortValuePredicate = null; + break; + case SHARDS: + preflightPredicate = null; + fromSortValuePredicate = filterByLongOffset(SnapshotInfo::totalShards, Integer.parseInt(fromSortValue), order); + break; + case FAILED_SHARDS: + preflightPredicate = null; + fromSortValuePredicate = filterByLongOffset(SnapshotInfo::failedShards, Integer.parseInt(fromSortValue), order); + break; + default: + throw new AssertionError("unexpected sort column [" + sortBy + "]"); + } + + if (snapshotPredicate == null) { + snapshotPredicate = fromSortValuePredicate; + } else if (fromSortValuePredicate != null) { + snapshotPredicate = fromSortValuePredicate.and(snapshotPredicate); + } + } + this.snapshotPredicate = snapshotPredicate; + } + + @Nullable + public Predicate snapshotPredicate() { + return snapshotPredicate; + } + + @Nullable + public BiPredicate preflightPredicate() { + return preflightPredicate; + } + + } + private static final class SnapshotsInRepo { private final List snapshotInfos;