From 452e27495dfae67fb43adc96b020fc1bbaa10961 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 2 Mar 2016 15:52:06 +0100 Subject: [PATCH 1/2] Handle eventally consistent blob lists in storage ITs --- .../gcloud/storage/it/ITStorageTest.java | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java index 8e954de57e68..1bdd14dd79f8 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.gcloud.Page; import com.google.gcloud.ReadChannel; @@ -287,8 +288,8 @@ public void testGetBlobFailNonExistingGeneration() { assertTrue(remoteBlob.delete()); } - @Test - public void testListBlobsSelectedFields() { + @Test(timeout = 5000) + public void testListBlobsSelectedFields() throws InterruptedException { String[] blobNames = {"test-list-blobs-selected-fields-blob1", "test-list-blobs-selected-fields-blob2"}; ImmutableMap metadata = ImmutableMap.of("k", "v"); @@ -307,10 +308,18 @@ public void testListBlobsSelectedFields() { Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), Storage.BlobListOption.fields(BlobField.METADATA)); - int index = 0; + // Listing blobs is eventually consistent, we loop until the list is of the expected size. The + // test fails if timeout is reached. + while (Iterators.size(page.values().iterator()) != 2) { + Thread.sleep(500); + page = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), + Storage.BlobListOption.fields(BlobField.METADATA)); + } + Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); for (Blob remoteBlob : page.values()) { assertEquals(BUCKET, remoteBlob.bucket()); - assertEquals(blobNames[index++], remoteBlob.name()); + assertTrue(blobSet.contains(remoteBlob.name())); assertEquals(metadata, remoteBlob.metadata()); assertNull(remoteBlob.contentType()); } @@ -318,8 +327,8 @@ public void testListBlobsSelectedFields() { assertTrue(remoteBlob2.delete()); } - @Test - public void testListBlobsEmptySelectedFields() { + @Test(timeout = 5000) + public void testListBlobsEmptySelectedFields() throws InterruptedException { String[] blobNames = {"test-list-blobs-empty-selected-fields-blob1", "test-list-blobs-empty-selected-fields-blob2"}; BlobInfo blob1 = BlobInfo.builder(BUCKET, blobNames[0]) @@ -335,17 +344,25 @@ public void testListBlobsEmptySelectedFields() { Page page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), Storage.BlobListOption.fields()); - int index = 0; + // Listing blobs is eventually consistent, we loop until the list is of the expected size. The + // test fails if timeout is reached. + while (Iterators.size(page.values().iterator()) != 2) { + Thread.sleep(500); + page = storage.list(BUCKET, + Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), + Storage.BlobListOption.fields()); + } + Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); for (Blob remoteBlob : page.values()) { assertEquals(BUCKET, remoteBlob.bucket()); - assertEquals(blobNames[index++], remoteBlob.name()); + assertTrue(blobSet.contains(remoteBlob.name())); assertNull(remoteBlob.contentType()); } assertTrue(remoteBlob1.delete()); assertTrue(remoteBlob2.delete()); } - @Test + @Test(timeout = 10000) public void testListBlobsVersioned() throws ExecutionException, InterruptedException { String bucketName = RemoteGcsHelper.generateBucketName(); Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build()); @@ -366,6 +383,14 @@ public void testListBlobsVersioned() throws ExecutionException, InterruptedExcep Page page = storage.list(bucketName, Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"), Storage.BlobListOption.versions(true)); + // Listing blobs is eventually consistent, we loop until the list is of the expected size. The + // test fails if timeout is reached. + while (Iterators.size(page.values().iterator()) != 3) { + Thread.sleep(500); + page = storage.list(bucketName, + Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"), + Storage.BlobListOption.versions(true)); + } Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); for (Blob remoteBlob : page.values()) { assertEquals(bucketName, remoteBlob.bucket()); From 622dcc9055f082b44ed32abfda9e585c7e76ce8e Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Thu, 3 Mar 2016 10:13:17 +0100 Subject: [PATCH 2/2] Replace values().iterator() with iterateAll() in blob list ITs --- .../gcloud/storage/it/ITStorageTest.java | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java index 1bdd14dd79f8..38521ae5e137 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/it/ITStorageTest.java @@ -101,13 +101,12 @@ public static void afterClass() throws ExecutionException, InterruptedException @Test(timeout = 5000) public void testListBuckets() throws InterruptedException { - Iterator bucketIterator = - storage.list(Storage.BucketListOption.prefix(BUCKET), - Storage.BucketListOption.fields()).values().iterator(); + Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), + Storage.BucketListOption.fields()).iterateAll(); while (!bucketIterator.hasNext()) { Thread.sleep(500); bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), - Storage.BucketListOption.fields()).values().iterator(); + Storage.BucketListOption.fields()).iterateAll(); } while (bucketIterator.hasNext()) { Bucket remoteBucket = bucketIterator.next(); @@ -310,14 +309,16 @@ public void testListBlobsSelectedFields() throws InterruptedException { Storage.BlobListOption.fields(BlobField.METADATA)); // Listing blobs is eventually consistent, we loop until the list is of the expected size. The // test fails if timeout is reached. - while (Iterators.size(page.values().iterator()) != 2) { + while (Iterators.size(page.iterateAll()) != 2) { Thread.sleep(500); page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-selected-fields-blob"), Storage.BlobListOption.fields(BlobField.METADATA)); } Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); - for (Blob remoteBlob : page.values()) { + Iterator iterator = page.iterateAll(); + while (iterator.hasNext()) { + Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.bucket()); assertTrue(blobSet.contains(remoteBlob.name())); assertEquals(metadata, remoteBlob.metadata()); @@ -346,14 +347,16 @@ public void testListBlobsEmptySelectedFields() throws InterruptedException { Storage.BlobListOption.fields()); // Listing blobs is eventually consistent, we loop until the list is of the expected size. The // test fails if timeout is reached. - while (Iterators.size(page.values().iterator()) != 2) { + while (Iterators.size(page.iterateAll()) != 2) { Thread.sleep(500); page = storage.list(BUCKET, Storage.BlobListOption.prefix("test-list-blobs-empty-selected-fields-blob"), Storage.BlobListOption.fields()); } Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); - for (Blob remoteBlob : page.values()) { + Iterator iterator = page.iterateAll(); + while (iterator.hasNext()) { + Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.bucket()); assertTrue(blobSet.contains(remoteBlob.name())); assertNull(remoteBlob.contentType()); @@ -362,7 +365,7 @@ public void testListBlobsEmptySelectedFields() throws InterruptedException { assertTrue(remoteBlob2.delete()); } - @Test(timeout = 10000) + @Test(timeout = 15000) public void testListBlobsVersioned() throws ExecutionException, InterruptedException { String bucketName = RemoteGcsHelper.generateBucketName(); Bucket bucket = storage.create(BucketInfo.builder(bucketName).versioningEnabled(true).build()); @@ -385,14 +388,16 @@ public void testListBlobsVersioned() throws ExecutionException, InterruptedExcep Storage.BlobListOption.versions(true)); // Listing blobs is eventually consistent, we loop until the list is of the expected size. The // test fails if timeout is reached. - while (Iterators.size(page.values().iterator()) != 3) { + while (Iterators.size(page.iterateAll()) != 3) { Thread.sleep(500); page = storage.list(bucketName, Storage.BlobListOption.prefix("test-list-blobs-versioned-blob"), Storage.BlobListOption.versions(true)); } Set blobSet = ImmutableSet.of(blobNames[0], blobNames[1]); - for (Blob remoteBlob : page.values()) { + Iterator iterator = page.iterateAll(); + while (iterator.hasNext()) { + Blob remoteBlob = iterator.next(); assertEquals(bucketName, remoteBlob.bucket()); assertTrue(blobSet.contains(remoteBlob.name())); assertNotNull(remoteBlob.generation());