From 1726c85ee7d3b1be58e6616de199d6cc2c01959f Mon Sep 17 00:00:00 2001 From: Gaole Meng Date: Wed, 29 Mar 2023 18:54:08 -0700 Subject: [PATCH 1/6] feat: add public api to stream writer to set the maximum wait time --- .../bigquery/storage/v1/ConnectionWorker.java | 2 +- .../bigquery/storage/v1/StreamWriter.java | 10 ++++++ .../bigquery/storage/v1/StreamWriterTest.java | 31 +++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java index 1aeb911943..12afbf13e0 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java @@ -75,7 +75,7 @@ class ConnectionWorker implements AutoCloseable { * We will constantly checking how much time we have been waiting for the next request callback * if we wait too much time we will start shutting down the connections and clean up the queues. */ - private static Duration MAXIMUM_REQUEST_CALLBACK_WAIT_TIME = Duration.ofMinutes(15); + static Duration MAXIMUM_REQUEST_CALLBACK_WAIT_TIME = Duration.ofMinutes(15); private Lock lock; private Condition hasMessageInWaitingQueue; diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java index b21a52a63d..bfa30c6141 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/StreamWriter.java @@ -518,6 +518,16 @@ public synchronized TableSchema getUpdatedSchema() { : null; } + /** + * Sets the maximum time a request is allowed to be waiting in request waiting queue. Under very + * low chance, it's possible for append request to be waiting indefintely for request callback + * when Google networking SDK does not detect the networking breakage. The default timeout is 15 + * minutes. We are investigating the root cause for callback not triggered by networking SDK. + */ + public static void setMaxRequestCallbackWaitTime(Duration waitTime) { + ConnectionWorker.MAXIMUM_REQUEST_CALLBACK_WAIT_TIME = waitTime; + } + long getCreationTimestamp() { return creationTimestamp; } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java index af36273102..bc6dd71690 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/StreamWriterTest.java @@ -15,6 +15,7 @@ */ package com.google.cloud.bigquery.storage.v1; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThrows; @@ -113,6 +114,7 @@ public StreamWriterTest() throws DescriptorValidationException {} @Before public void setUp() throws Exception { testBigQueryWrite = new FakeBigQueryWrite(); + StreamWriter.setMaxRequestCallbackWaitTime(java.time.Duration.ofSeconds(10000)); ConnectionWorker.setMaxInflightQueueWaitTime(300000); serviceHelper = new MockServiceHelper( @@ -947,6 +949,35 @@ public void testMessageTooLarge() throws Exception { writer.close(); } + @Test + public void testThrowExceptionWhileWithinAppendLoop_MaxWaitTimeExceed() throws Exception { + ProtoSchema schema1 = createProtoSchema("foo"); + StreamWriter.setMaxRequestCallbackWaitTime(java.time.Duration.ofSeconds(1)); + StreamWriter writer = + StreamWriter.newBuilder(TEST_STREAM_1, client).setWriterSchema(schema1).build(); + testBigQueryWrite.setResponseSleep(org.threeten.bp.Duration.ofSeconds(3)); + + long appendCount = 10; + for (int i = 0; i < appendCount; i++) { + testBigQueryWrite.addResponse(createAppendResponse(i)); + } + + // In total insert 5 requests, + List> futures = new ArrayList<>(); + for (int i = 0; i < appendCount; i++) { + futures.add(writer.append(createProtoRows(new String[] {String.valueOf(i)}), i)); + } + + for (int i = 0; i < appendCount; i++) { + int finalI = i; + ExecutionException ex = + assertThrows( + ExecutionException.class, + () -> futures.get(finalI).get().getAppendResult().getOffset().getValue()); + assertThat(ex.getCause()).hasMessageThat().contains("Request has waited in inflight queue"); + } + } + @Test public void testAppendWithResetSuccess() throws Exception { try (StreamWriter writer = getTestStreamWriter()) { From 3f0c5ebe7f400cb49aedbe329165725eda5f5329 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Thu, 30 Mar 2023 20:47:37 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 145a863499..76e80afa7f 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquerystorage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquerystorage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.1: [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From a448c8b205ea41796b77e5f644ce527f0db53501 Mon Sep 17 00:00:00 2001 From: Gaole Meng Date: Thu, 30 Mar 2023 14:51:35 -0700 Subject: [PATCH 3/6] modify back the readme change from owl post processor --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 76e80afa7f..145a863499 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquerystorage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquerystorage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.1: +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From b15b140d703fe510861cda925aece31ed7c3b234 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Fri, 31 Mar 2023 20:18:30 +0000 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 145a863499..5eaba81994 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-bigquerystorage' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-bigquerystorage:2.34.1' +implementation 'com.google.cloud:google-cloud-bigquerystorage:2.34.2' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.34.1" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquerystorage" % "2.34.2" ``` @@ -219,7 +219,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-bigquerystorage/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-bigquerystorage.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.1 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-bigquerystorage/2.34.2 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles From affa11ac873b38d27b6f36d5a8501e70105ba68b Mon Sep 17 00:00:00 2001 From: Gaole Meng Date: Fri, 21 Apr 2023 17:49:54 -0700 Subject: [PATCH 5/6] fix: Reduce the timeout to 5 minutes for the requests wait time in queue. Since in write api server side we have total timeout of 2 minutes, it does not make sense to wait 15 minutes to determine whether we have met dead connection, let's reduce the timeout here --- .../google/cloud/bigquery/storage/v1/ConnectionWorker.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java index 12afbf13e0..64abf82bb9 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/ConnectionWorker.java @@ -75,7 +75,7 @@ class ConnectionWorker implements AutoCloseable { * We will constantly checking how much time we have been waiting for the next request callback * if we wait too much time we will start shutting down the connections and clean up the queues. */ - static Duration MAXIMUM_REQUEST_CALLBACK_WAIT_TIME = Duration.ofMinutes(15); + static Duration MAXIMUM_REQUEST_CALLBACK_WAIT_TIME = Duration.ofMinutes(5); private Lock lock; private Condition hasMessageInWaitingQueue; @@ -321,7 +321,7 @@ public void run() { } private void resetConnection() { - log.info("Reconnecting for stream:" + streamName + " id: " + writerId); + log.info("Start connecting stream: " + streamName + " id: " + writerId); if (this.streamConnection != null) { // It's safe to directly close the previous connection as the in flight messages // will be picked up by the next connection. @@ -344,7 +344,7 @@ public void run(Throwable finalStatus) { doneCallback(finalStatus); } }); - log.info("Reconnect done for stream:" + streamName + " id: " + writerId); + log.info("Finish connecting stream: " + streamName + " id: " + writerId); } /** Schedules the writing of rows at given offset. */ From 60869f74065fc100f34f80ba00ee816343f7ba01 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 25 Apr 2023 00:18:28 +0000 Subject: [PATCH 6/6] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot=20po?= =?UTF-8?q?st-processor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e453f63b9a..d4392b52df 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ If you are using Maven without the BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies: ```Groovy -implementation platform('com.google.cloud:libraries-bom:26.12.0') +implementation platform('com.google.cloud:libraries-bom:26.13.0') implementation 'com.google.cloud:google-cloud-bigquerystorage' ```