diff --git a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java index b83b55f224242..6f5bcdfc8dfaf 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/snapshots/SnapshotStressTestsIT.java @@ -809,9 +809,33 @@ private void startCleaner() { Releasables.close(releaseAll); logger.info("--> completed cleanup of [{}]", trackedRepository.repositoryName); final RepositoryCleanupResult result = cleanupRepositoryResponse.result(); - assertThat(Strings.toString(result), result.blobs(), equalTo(0L)); - assertThat(Strings.toString(result), result.bytes(), equalTo(0L)); - startCleaner(); + if (result.bytes() > 0L || result.blobs() > 0L) { + // we could legitimately run into dangling blobs as the result of a shard snapshot failing half-way + // through the snapshot because of a concurrent index-close or -delete. The second round of cleanup on + // the same repository however should always find no more dangling blobs and be a no-op since we block all + // concurrent operations on the repository. + client.admin() + .cluster() + .prepareCleanupRepository(trackedRepository.repositoryName) + .execute(mustSucceed(secondCleanupRepositoryResponse -> { + final RepositoryCleanupResult secondCleanupResult = secondCleanupRepositoryResponse.result(); + if (secondCleanupResult.blobs() == 1) { + // The previous cleanup actually leaves behind a stale index-N blob, so this cleanup removes it + // and reports it in its response. When https://github.com/elastic/elasticsearch/pull/100718 is + // fixed the second cleanup will be a proper no-op and we can remove this lenience -- TODO + } else { + assertThat(Strings.toString(secondCleanupResult), secondCleanupResult.blobs(), equalTo(0L)); + assertThat(Strings.toString(secondCleanupResult), secondCleanupResult.bytes(), equalTo(0L)); + } + Releasables.close(releaseAll); + logger.info("--> completed second cleanup of [{}]", trackedRepository.repositoryName); + startCleaner(); + })); + } else { + Releasables.close(releaseAll); + logger.info("--> completed cleanup of [{}]", trackedRepository.repositoryName); + startCleaner(); + } })); startedCleanup = true;