From 8da1edf6e1e68da58530bcfb74cc7f0ed7f30029 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 11 Jun 2019 13:42:22 -0700 Subject: [PATCH 1/4] vision: address flakes due to collisions. Multiple tests in the vision suite write output to GCS. Sadly, all wrote to the same output prefix in a nondeterministic order, which can cause issues. This change does two things: 1) ensures each test run uses a unique prefix 2) makes the prefix clearer to disambiguate output from the cloud_client and the beta samples. I suspect the beta samples should be cleaned up, but it appears there's still linkages into cloud docs which is a more involved change. This should get the tests green once more. --- .../src/test/java/com/example/vision/DetectIT.java | 3 ++- .../src/test/java/com/example/vision/DetectIT.java | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java b/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java index 65d492fe5d1..d70e02743b3 100644 --- a/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java +++ b/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java @@ -21,6 +21,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -35,7 +36,7 @@ public class DetectIT { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); // private static final String BUCKET = PROJECT_ID; private static final String BUCKET = "java-docs-samples-testing"; - private static final String OUTPUT_PREFIX = "OCR_PDF_TEST_OUTPUT"; + private static final String OUTPUT_PREFIX = "OCR_PDF_TEST_OUTPUT_VISION_BETA_" + UUID.randomUUID().toString(); private ByteArrayOutputStream bout; private PrintStream out; private Detect app; diff --git a/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java b/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java index c0317744a5c..3f4857d9e63 100644 --- a/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java +++ b/vision/cloud-client/src/test/java/com/example/vision/DetectIT.java @@ -26,6 +26,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; +import java.util.UUID; import org.junit.After; import org.junit.Before; @@ -42,7 +43,7 @@ public class DetectIT { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); private static final String ASSET_BUCKET = "cloud-samples-data"; private static final String OUTPUT_BUCKET = PROJECT_ID; - private static final String OUTPUT_PREFIX = "OCR_PDF_TEST_OUTPUT"; + private static final String OUTPUT_PREFIX = "OCR_PDF_TEST_OUTPUT_" + UUID.randomUUID().toString(); @Before public void setUp() throws IOException { @@ -363,13 +364,13 @@ public void testDetectDocumentsGcs() throws Exception { // Assert String got = bout.toString(); + assertThat(got).contains("OIL, GAS AND MINERAL LEASE"); Storage storage = StorageOptions.getDefaultInstance().getService(); Page blobs = storage.list(OUTPUT_BUCKET, BlobListOption.currentDirectory(), BlobListOption.prefix(OUTPUT_PREFIX + "/")); - for (Blob blob : blobs.iterateAll()) { blob.delete(); } @@ -397,4 +398,4 @@ public void testDetectLocalizedObjectsGcs() throws Exception { String got = bout.toString().toLowerCase(); assertThat(got).contains("dog"); } -} \ No newline at end of file +} From 98223deb19eef537edd0ddf13342ddc2d745586f Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 11 Jun 2019 14:41:22 -0700 Subject: [PATCH 2/4] beta: convert batch future to actually wait for output properly. --- .../vision/AsyncBatchAnnotateImagesGcs.java | 21 +++++++------------ .../java/com/example/vision/DetectIT.java | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java index 8454227621b..ecab9a42341 100644 --- a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java +++ b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java @@ -18,23 +18,16 @@ // [START vision_async_batch_annotate_images_beta] import com.google.api.core.ApiFuture; +import com.google.api.gax.longrunning.OperationFuture; import com.google.api.gax.paging.Page; import com.google.cloud.storage.Blob; import com.google.cloud.storage.Bucket; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobListOption; import com.google.cloud.storage.StorageOptions; -import com.google.cloud.vision.v1p4beta1.AnnotateImageRequest; -import com.google.cloud.vision.v1p4beta1.AsyncBatchAnnotateImagesRequest; +import com.google.cloud.vision.v1p4beta1.*; import com.google.cloud.vision.v1p4beta1.BatchAnnotateImagesResponse.Builder; -import com.google.cloud.vision.v1p4beta1.BatchAnnotateImagesResponse; -import com.google.cloud.vision.v1p4beta1.Feature; import com.google.cloud.vision.v1p4beta1.Feature.Type; -import com.google.cloud.vision.v1p4beta1.GcsDestination; -import com.google.cloud.vision.v1p4beta1.Image; -import com.google.cloud.vision.v1p4beta1.ImageAnnotatorClient; -import com.google.cloud.vision.v1p4beta1.ImageSource; -import com.google.cloud.vision.v1p4beta1.OutputConfig; import com.google.longrunning.Operation; import com.google.protobuf.util.JsonFormat; @@ -51,7 +44,6 @@ public static void asyncBatchAnnotateImagesGcs(String gcsSourcePath, String gcsD throws Exception { // String gcsSourcePath = "gs://YOUR_BUCKET_ID/path_to_your_data"; // String gcsDestinationPath = "gs://YOUR_BUCKET_ID/path_to_store_annotation"; - try (ImageAnnotatorClient client = ImageAnnotatorClient.create()) { List requests = new ArrayList<>(); @@ -87,13 +79,14 @@ public static void asyncBatchAnnotateImagesGcs(String gcsSourcePath, String gcsD .setOutputConfig(outputConfig) .build(); - ApiFuture future = client.asyncBatchAnnotateImagesCallable().futureCall(request); - // Wait for the request to finish. (The result is not used, since the API saves the result to - // the specified location on GCS.) - Operation response = future.get(180, TimeUnit.SECONDS); + OperationFuture response = + client.asyncBatchAnnotateImagesAsync(request); System.out.println("Waiting for the operation to finish."); + // we're not processing the response, since we'll be reading the output from GCS. + response.get(180, TimeUnit.SECONDS); + // Once the request has completed and the output has been // written to GCS, we can list all the output files. Storage storage = StorageOptions.getDefaultInstance().getService(); diff --git a/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java b/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java index d70e02743b3..544c6f93173 100644 --- a/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java +++ b/vision/beta/cloud-client/src/test/java/com/example/vision/DetectIT.java @@ -34,9 +34,9 @@ @SuppressWarnings("checkstyle:abbreviationaswordinname") public class DetectIT { private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); - // private static final String BUCKET = PROJECT_ID; private static final String BUCKET = "java-docs-samples-testing"; - private static final String OUTPUT_PREFIX = "OCR_PDF_TEST_OUTPUT_VISION_BETA_" + UUID.randomUUID().toString(); + private static final String OUTPUT_BUCKET = PROJECT_ID; + private static final String OUTPUT_PREFIX = "OUTPUT_VISION_BETA_" + UUID.randomUUID().toString(); private ByteArrayOutputStream bout; private PrintStream out; private Detect app; @@ -126,7 +126,7 @@ public void testAsyncBatchAnnotateImagesGcs() throws Exception { // Act AsyncBatchAnnotateImagesGcs.asyncBatchAnnotateImagesGcs( "gs://cloud-samples-data/vision/label/wakeupcat.jpg", - "gs://" + BUCKET + "/" + OUTPUT_PREFIX + "/"); + "gs://" + OUTPUT_BUCKET + "/" + OUTPUT_PREFIX + "/"); // Assert String got = bout.toString(); From 01ed1575737de82e1e02a37ab110f72f6e20b2fe Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 11 Jun 2019 15:34:45 -0700 Subject: [PATCH 3/4] explicit imports --- .../vision/AsyncBatchAnnotateImagesGcs.java | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java index ecab9a42341..a9a3848abf1 100644 --- a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java +++ b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java @@ -17,7 +17,6 @@ package com.example.vision; // [START vision_async_batch_annotate_images_beta] -import com.google.api.core.ApiFuture; import com.google.api.gax.longrunning.OperationFuture; import com.google.api.gax.paging.Page; import com.google.cloud.storage.Blob; @@ -25,11 +24,18 @@ import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobListOption; import com.google.cloud.storage.StorageOptions; -import com.google.cloud.vision.v1p4beta1.*; +import com.google.cloud.vision.v1p4beta1.AnnotateImageRequest; +import com.google.cloud.vision.v1p4beta1.AsyncBatchAnnotateImagesRequest; +import com.google.cloud.vision.v1p4beta1.AsyncBatchAnnotateImagesResponse; import com.google.cloud.vision.v1p4beta1.BatchAnnotateImagesResponse.Builder; +import com.google.cloud.vision.v1p4beta1.Feature; import com.google.cloud.vision.v1p4beta1.Feature.Type; - -import com.google.longrunning.Operation; +import com.google.cloud.vision.v1p4beta1.GcsDestination; +import com.google.cloud.vision.v1p4beta1.Image; +import com.google.cloud.vision.v1p4beta1.ImageAnnotatorClient; +import com.google.cloud.vision.v1p4beta1.ImageSource; +import com.google.cloud.vision.v1p4beta1.OperationMetadata; +import com.google.cloud.vision.v1p4beta1.OutputConfig; import com.google.protobuf.util.JsonFormat; import java.util.ArrayList; import java.util.List; From d6c945cf511fa47bbf7820d750f9de4604de0762 Mon Sep 17 00:00:00 2001 From: Seth Hollyman Date: Tue, 11 Jun 2019 16:13:39 -0700 Subject: [PATCH 4/4] additional explicit import --- .../java/com/example/vision/AsyncBatchAnnotateImagesGcs.java | 1 + 1 file changed, 1 insertion(+) diff --git a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java index a9a3848abf1..5fc863ffde3 100644 --- a/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java +++ b/vision/beta/cloud-client/src/main/java/com/example/vision/AsyncBatchAnnotateImagesGcs.java @@ -27,6 +27,7 @@ import com.google.cloud.vision.v1p4beta1.AnnotateImageRequest; import com.google.cloud.vision.v1p4beta1.AsyncBatchAnnotateImagesRequest; import com.google.cloud.vision.v1p4beta1.AsyncBatchAnnotateImagesResponse; +import com.google.cloud.vision.v1p4beta1.BatchAnnotateImagesResponse; import com.google.cloud.vision.v1p4beta1.BatchAnnotateImagesResponse.Builder; import com.google.cloud.vision.v1p4beta1.Feature; import com.google.cloud.vision.v1p4beta1.Feature.Type;