From 16c606c6b6373041e4adf3b57ce05189d25662da Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Wed, 1 Dec 2021 14:07:54 -0500 Subject: [PATCH 1/8] retrying job create with new job id each time --- google-cloud-bigquery/.flattened-pom.xml | 805 ++++++++++++++++++ .../google/cloud/bigquery/BigQueryImpl.java | 21 +- .../bigquery/BigQueryRetryAlgorithm.java | 46 +- .../cloud/bigquery/BigQueryRetryHelper.java | 4 +- 4 files changed, 862 insertions(+), 14 deletions(-) create mode 100644 google-cloud-bigquery/.flattened-pom.xml diff --git a/google-cloud-bigquery/.flattened-pom.xml b/google-cloud-bigquery/.flattened-pom.xml new file mode 100644 index 000000000..080e34c23 --- /dev/null +++ b/google-cloud-bigquery/.flattened-pom.xml @@ -0,0 +1,805 @@ + + + 4.0.0 + com.google.cloud + google-cloud-bigquery + 2.4.2-SNAPSHOT + BigQuery + BigQuery + https://github.com/googleapis/java-bigquery + + Google LLC + + + + Apache-2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + + + chingor + Jeff Ching + chingor@google.com + Google + + Developer + + + + + scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery + scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery + https://github.com/googleapis/java-bigquery/google-cloud-bigquery + + + GitHub Issues + https://github.com/googleapis/java-bigquery/issues + + + + sonatype-nexus-staging + https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + sonatype-nexus-snapshots + https://google.oss.sonatype.org/content/repositories/snapshots + + + + + com.google.cloud + google-cloud-core + 2.3.0 + compile + + + guava + com.google.guava + + + gax + com.google.api + + + auto-value-annotations + com.google.auto.value + + + protobuf-java-util + com.google.protobuf + + + proto-google-common-protos + com.google.api.grpc + + + proto-google-iam-v1 + com.google.api.grpc + + + threetenbp + org.threeten + + + api-common + com.google.api + + + google-auth-library-credentials + com.google.auth + + + google-auth-library-oauth2-http + com.google.auth + + + google-http-client + com.google.http-client + + + google-http-client-gson + com.google.http-client + + + protobuf-java + com.google.protobuf + + + jsr305 + com.google.code.findbugs + + + false + + + com.google.protobuf + protobuf-java-util + 3.19.1 + compile + + + protobuf-java + com.google.protobuf + + + guava + com.google.guava + + + error_prone_annotations + com.google.errorprone + + + j2objc-annotations + com.google.j2objc + + + jsr305 + com.google.code.findbugs + + + gson + com.google.code.gson + + + false + + + com.google.code.gson + gson + 2.8.9 + compile + false + + + com.google.api.grpc + proto-google-common-protos + 2.6.0 + compile + + + protobuf-java + com.google.protobuf + + + false + + + com.google.api.grpc + proto-google-iam-v1 + 1.1.7 + compile + + + protobuf-java + com.google.protobuf + + + proto-google-common-protos + com.google.api.grpc + + + false + + + com.google.auth + google-auth-library-credentials + 1.3.0 + compile + false + + + com.google.http-client + google-http-client-gson + 1.40.1 + compile + + + google-http-client + com.google.http-client + + + gson + com.google.code.gson + + + false + + + com.google.protobuf + protobuf-java + 3.19.1 + compile + false + + + com.google.cloud + google-cloud-core-http + 2.3.0 + compile + + + google-cloud-core + com.google.cloud + + + google-auth-library-credentials + com.google.auth + + + google-auth-library-oauth2-http + com.google.auth + + + google-http-client + com.google.http-client + + + guava + com.google.guava + + + google-api-client + com.google.api-client + + + google-http-client-appengine + com.google.http-client + + + gax + com.google.api + + + gax-httpjson + com.google.api + + + jsr305 + com.google.code.findbugs + + + opencensus-api + io.opencensus + + + opencensus-contrib-http-util + io.opencensus + + + api-common + com.google.api + + + false + + + com.google.api-client + google-api-client + 1.32.2 + compile + + + google-oauth-client + com.google.oauth-client + + + google-http-client-gson + com.google.http-client + + + guava + com.google.guava + + + google-http-client-apache-v2 + com.google.http-client + + + httpcore + org.apache.httpcomponents + + + httpclient + org.apache.httpcomponents + + + google-http-client + com.google.http-client + + + false + + + com.google.oauth-client + google-oauth-client + 1.32.1 + compile + + + google-http-client + com.google.http-client + + + guava + com.google.guava + + + false + + + com.google.http-client + google-http-client-apache-v2 + 1.40.1 + compile + + + google-http-client + com.google.http-client + + + httpclient + org.apache.httpcomponents + + + httpcore + org.apache.httpcomponents + + + false + + + com.google.http-client + google-http-client-appengine + 1.40.1 + compile + + + google-http-client + com.google.http-client + + + appengine-api-1.0-sdk + com.google.appengine + + + false + + + com.google.api + gax-httpjson + 0.92.0 + compile + + + gax + com.google.api + + + api-common + com.google.api + + + proto-google-common-protos + com.google.api.grpc + + + google-auth-library-credentials + com.google.auth + + + gson + com.google.code.gson + + + guava + com.google.guava + + + google-http-client + com.google.http-client + + + protobuf-java + com.google.protobuf + + + threetenbp + org.threeten + + + google-auth-library-oauth2-http + com.google.auth + + + jsr305 + com.google.code.findbugs + + + google-http-client-gson + com.google.http-client + + + protobuf-java-util + com.google.protobuf + + + false + + + io.opencensus + opencensus-api + 0.28.0 + compile + + + grpc-context + io.grpc + + + false + + + io.grpc + grpc-context + 1.42.1 + compile + false + + + io.opencensus + opencensus-contrib-http-util + 0.28.0 + compile + + + opencensus-api + io.opencensus + + + guava + com.google.guava + + + false + + + com.google.http-client + google-http-client-jackson2 + 1.40.1 + compile + + + google-http-client + com.google.http-client + + + jackson-core + com.fasterxml.jackson.core + + + false + + + com.fasterxml.jackson.core + jackson-core + 2.13.0 + compile + false + + + javax.annotation + javax.annotation-api + 1.3.2 + compile + false + + + com.google.auto.value + auto-value-annotations + 1.8.2 + provided + false + + + com.google.http-client + google-http-client + 1.40.1 + compile + + + httpclient + org.apache.httpcomponents + + + httpcore + org.apache.httpcomponents + + + jsr305 + com.google.code.findbugs + + + guava + com.google.guava + + + j2objc-annotations + com.google.j2objc + + + opencensus-api + io.opencensus + + + opencensus-contrib-http-util + io.opencensus + + + false + + + org.apache.httpcomponents + httpclient + 4.5.13 + compile + + + httpcore + org.apache.httpcomponents + + + commons-logging + commons-logging + + + commons-codec + commons-codec + + + false + + + commons-logging + commons-logging + 1.2 + compile + + + log4j + log4j + + + logkit + logkit + + + avalon-framework + avalon-framework + + + servlet-api + javax.servlet + + + false + + + commons-codec + commons-codec + 1.15 + compile + false + + + org.apache.httpcomponents + httpcore + 4.4.14 + compile + false + + + com.google.j2objc + j2objc-annotations + 1.3 + compile + false + + + org.checkerframework + checker-compat-qual + 2.5.5 + compile + false + + + com.google.auth + google-auth-library-oauth2-http + 1.3.0 + compile + + + auto-value-annotations + com.google.auto.value + + + jsr305 + com.google.code.findbugs + + + google-auth-library-credentials + com.google.auth + + + google-http-client + com.google.http-client + + + google-http-client-gson + com.google.http-client + + + guava + com.google.guava + + + false + + + com.google.code.findbugs + jsr305 + 3.0.2 + compile + false + + + com.google.apis + google-api-services-bigquery + v2-rev20211106-1.32.1 + compile + + + google-api-client + com.google.api-client + + + false + + + com.google.api + api-common + 2.1.0 + compile + + + guava + com.google.guava + + + jsr305 + com.google.code.findbugs + + + javax.annotation-api + javax.annotation + + + auto-value-annotations + com.google.auto.value + + + false + + + com.google.guava + guava + 31.0.1-jre + compile + + + failureaccess + com.google.guava + + + listenablefuture + com.google.guava + + + jsr305 + com.google.code.findbugs + + + checker-qual + org.checkerframework + + + error_prone_annotations + com.google.errorprone + + + j2objc-annotations + com.google.j2objc + + + srczip + jdk + + + false + + + com.google.guava + failureaccess + 1.0.1 + compile + false + + + com.google.guava + listenablefuture + 9999.0-empty-to-avoid-conflict-with-guava + compile + false + + + org.checkerframework + checker-qual + 3.19.0 + compile + false + + + com.google.api + gax + 2.7.0 + compile + + + api-common + com.google.api + + + google-auth-library-credentials + com.google.auth + + + threetenbp + org.threeten + + + google-auth-library-oauth2-http + com.google.auth + + + jsr305 + com.google.code.findbugs + + + guava + com.google.guava + + + opencensus-api + io.opencensus + + + false + + + org.threeten + threetenbp + 1.5.2 + compile + false + + + com.google.errorprone + error_prone_annotations + 2.10.0 + compile + false + + + diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 7d0e4c555..4af9a8789 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -363,21 +363,32 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { // This makes it difficult to translate without throwing. // Fixing this entails some work on BaseServiceException.translate. // Since that affects a bunch of APIs, we should fix this as a separate change. + final boolean idRandomCopy = idRandom; + final JobInfo jobInfoCopy = jobInfo; try { try { return Job.fromPb( this, - runWithRetries( + BigQueryRetryHelper.runWithRetries( new Callable() { @Override public com.google.api.services.bigquery.model.Job call() { - return bigQueryRpc.create(jobPb, optionsMap); + if(idRandomCopy){ + JobInfo newJobInfo = jobInfoCopy.toBuilder().setJobId(idProvider.get()).build(); + com.google.api.services.bigquery.model.Job newJobPb = + newJobInfo.setProjectId(getOptions().getProjectId()).toPb(); + return bigQueryRpc.create(newJobPb, optionsMap); + } + else { + return bigQueryRpc.create(jobPb, optionsMap); + } } }, getOptions().getRetrySettings(), - BigQueryBaseService.BIGQUERY_EXCEPTION_HANDLER, - getOptions().getClock())); - } catch (RetryHelper.RetryHelperException e) { + EXCEPTION_HANDLER, + getOptions().getClock(), + DEFAULT_RETRY_CONFIG)); + } catch (BigQueryRetryHelper.BigQueryRetryHelperException e) { throw BigQueryException.translateAndThrow(e); } } catch (BigQueryException e) { diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java index a4e642191..4c36523f8 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java @@ -72,12 +72,12 @@ public boolean shouldRetry( // the exception messages boolean shouldRetry = (shouldRetryBasedOnResult(context, previousThrowable, previousResponse) - || shouldRetryBasedOnBigQueryRetryConfig(previousThrowable, bigQueryRetryConfig)) + || shouldRetryBasedOnBigQueryRetryConfig(previousThrowable, bigQueryRetryConfig, previousResponse)) && shouldRetryBasedOnTiming(context, nextAttemptSettings); - if (LOG.isLoggable(Level.FINEST)) { + if (LOG.isLoggable(Level.INFO)) { LOG.log( - Level.FINEST, + Level.INFO, "Retrying with:\n{0}\n{1}\n{2}\n{3}\n{4}\n{5}", new Object[] { "BigQuery attemptCount: " + attemptCount, @@ -92,13 +92,19 @@ public boolean shouldRetry( } private boolean shouldRetryBasedOnBigQueryRetryConfig( - Throwable previousThrowable, BigQueryRetryConfig bigQueryRetryConfig) { + Throwable previousThrowable, BigQueryRetryConfig bigQueryRetryConfig, ResponseT previousResponse) { /* We are deciding if a given error should be retried on the basis of error message. Cannot rely on Error/Status code as for example error code 400 (which is not retriable) could be thrown due to rateLimitExceed, which is retriable */ - String errorDesc; - if (previousThrowable != null && (errorDesc = previousThrowable.getMessage()) != null) { + String errorDesc = null; + if (previousThrowable != null ){ + errorDesc = previousThrowable.getMessage(); + } else if (previousResponse != null){ + errorDesc = previousResponse.toString(); + } + + if (errorDesc != null){ errorDesc = errorDesc.toLowerCase(); // for case insensitive comparison for (Iterator retriableMessages = bigQueryRetryConfig.getRetriableErrorMessages().iterator(); @@ -120,6 +126,31 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( } } } + // status code 200 + /* + if (previousResponse != null && (errorDesc = previousResponse.toString()) != null) { + errorDesc = errorDesc.toLowerCase(); // for case insensitive comparison + for (Iterator retriableMessages = + bigQueryRetryConfig.getRetriableErrorMessages().iterator(); + retriableMessages.hasNext(); ) { + if (errorDesc.contains( + retriableMessages + .next() + .toLowerCase())) { // Error message should be retried, implementing cases + // insensitive match + return true; + } + } + + // Check if there's a regex which matches the error message. This avoids too many regex + // matches which is expensive + for (Iterator retriableRegExes = bigQueryRetryConfig.getRetriableRegExes().iterator(); + retriableRegExes.hasNext(); ) { + if (matchRegEx(retriableRegExes.next(), errorDesc)) { + return true; + } + } + }*/ return false; } @@ -161,7 +192,8 @@ public TimedAttemptSettings createNextAttempt( if (!((shouldRetryBasedOnResult(context, previousThrowable, previousResponse) || shouldRetryBasedOnBigQueryRetryConfig( previousThrowable, - bigQueryRetryConfig)))) { // Calling shouldRetryBasedOnBigQueryRetryConfig to check if + bigQueryRetryConfig, + previousResponse)))) { // Calling shouldRetryBasedOnBigQueryRetryConfig to check if // the error message could be retried return null; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java index 405a2371e..74a4c7141 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java @@ -72,9 +72,9 @@ private static V run( RetryingExecutor executor = new DirectRetryingExecutor<>(retryAlgorithm); // Log retry info - if (LOG.isLoggable(Level.FINEST)) { + if (LOG.isLoggable(Level.INFO)) { LOG.log( - Level.FINEST, + Level.INFO, "Retrying with:\n{0}\n{1}", new Object[] { "BigQuery retried method: " + callable.getClass().getEnclosingMethod().getName(), From af20e7e097081a6b2474f7c5a505f6cf64cc12e3 Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Fri, 3 Dec 2021 08:48:44 -0500 Subject: [PATCH 2/8] retrying for rate limit exceeded on create Job regardless of the status code 200 --- .../google/cloud/bigquery/BigQueryImpl.java | 20 +++++++++---------- .../bigquery/BigQueryRetryAlgorithm.java | 9 +++++++-- .../cloud/bigquery/BigQueryRetryHelper.java | 4 ++-- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 4af9a8789..8a5619898 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -347,15 +347,9 @@ public JobId get() { @InternalApi("visible for testing") Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { - boolean idRandom = false; - if (jobInfo.getJobId() == null) { - jobInfo = jobInfo.toBuilder().setJobId(idProvider.get()).build(); - idRandom = true; - } - final com.google.api.services.bigquery.model.Job jobPb = - jobInfo.setProjectId(getOptions().getProjectId()).toPb(); - final Map optionsMap = optionMap(options); + final boolean idRandom = (jobInfo.getJobId() == null); + final Map optionsMap = optionMap(options); BigQueryException createException; // NOTE(pongad): This double-try structure is admittedly odd. // translateAndThrow itself throws, and pretends to return an exception only @@ -363,8 +357,8 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { // This makes it difficult to translate without throwing. // Fixing this entails some work on BaseServiceException.translate. // Since that affects a bunch of APIs, we should fix this as a separate change. - final boolean idRandomCopy = idRandom; final JobInfo jobInfoCopy = jobInfo; + final JobId[] finalJobId = new JobId[1]; try { try { return Job.fromPb( @@ -373,13 +367,17 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { new Callable() { @Override public com.google.api.services.bigquery.model.Job call() { - if(idRandomCopy){ + if(idRandom){ + // generate new random job JobInfo newJobInfo = jobInfoCopy.toBuilder().setJobId(idProvider.get()).build(); com.google.api.services.bigquery.model.Job newJobPb = newJobInfo.setProjectId(getOptions().getProjectId()).toPb(); + finalJobId[0] = newJobInfo.getJobId(); return bigQueryRpc.create(newJobPb, optionsMap); } else { + com.google.api.services.bigquery.model.Job jobPb = + jobInfo.setProjectId(getOptions().getProjectId()).toPb(); return bigQueryRpc.create(jobPb, optionsMap); } } @@ -407,7 +405,7 @@ public com.google.api.services.bigquery.model.Job call() { // fetch a job created by someone else. Job job; try { - job = getJob(jobInfo.getJobId()); + job = getJob(finalJobId[0]); } catch (BigQueryException e) { throw createException; } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java index 4c36523f8..bba0aaecb 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java @@ -75,9 +75,9 @@ public boolean shouldRetry( || shouldRetryBasedOnBigQueryRetryConfig(previousThrowable, bigQueryRetryConfig, previousResponse)) && shouldRetryBasedOnTiming(context, nextAttemptSettings); - if (LOG.isLoggable(Level.INFO)) { + if (LOG.isLoggable(Level.FINEST)) { LOG.log( - Level.INFO, + Level.FINEST, "Retrying with:\n{0}\n{1}\n{2}\n{3}\n{4}\n{5}", new Object[] { "BigQuery attemptCount: " + attemptCount, @@ -101,6 +101,11 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( if (previousThrowable != null ){ errorDesc = previousThrowable.getMessage(); } else if (previousResponse != null){ + /* + In some cases error messages may come without an exception + e.g. status code 200 with a rate limit exceeded for job create + in these cases there is now previousThrowable so we need to check previousResponse + */ errorDesc = previousResponse.toString(); } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java index 74a4c7141..405a2371e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryHelper.java @@ -72,9 +72,9 @@ private static V run( RetryingExecutor executor = new DirectRetryingExecutor<>(retryAlgorithm); // Log retry info - if (LOG.isLoggable(Level.INFO)) { + if (LOG.isLoggable(Level.FINEST)) { LOG.log( - Level.INFO, + Level.FINEST, "Retrying with:\n{0}\n{1}", new Object[] { "BigQuery retried method: " + callable.getClass().getEnclosingMethod().getName(), From 124116bd0a69ebd2bef31c77c2a3e0a652677db8 Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Tue, 7 Dec 2021 14:28:29 -0500 Subject: [PATCH 3/8] removing flattened pom --- google-cloud-bigquery/.flattened-pom.xml | 805 ----------------------- 1 file changed, 805 deletions(-) delete mode 100644 google-cloud-bigquery/.flattened-pom.xml diff --git a/google-cloud-bigquery/.flattened-pom.xml b/google-cloud-bigquery/.flattened-pom.xml deleted file mode 100644 index 080e34c23..000000000 --- a/google-cloud-bigquery/.flattened-pom.xml +++ /dev/null @@ -1,805 +0,0 @@ - - - 4.0.0 - com.google.cloud - google-cloud-bigquery - 2.4.2-SNAPSHOT - BigQuery - BigQuery - https://github.com/googleapis/java-bigquery - - Google LLC - - - - Apache-2.0 - https://www.apache.org/licenses/LICENSE-2.0.txt - - - - - chingor - Jeff Ching - chingor@google.com - Google - - Developer - - - - - scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery - scm:git:git@github.com:googleapis/java-bigquery.git/google-cloud-bigquery - https://github.com/googleapis/java-bigquery/google-cloud-bigquery - - - GitHub Issues - https://github.com/googleapis/java-bigquery/issues - - - - sonatype-nexus-staging - https://google.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - sonatype-nexus-snapshots - https://google.oss.sonatype.org/content/repositories/snapshots - - - - - com.google.cloud - google-cloud-core - 2.3.0 - compile - - - guava - com.google.guava - - - gax - com.google.api - - - auto-value-annotations - com.google.auto.value - - - protobuf-java-util - com.google.protobuf - - - proto-google-common-protos - com.google.api.grpc - - - proto-google-iam-v1 - com.google.api.grpc - - - threetenbp - org.threeten - - - api-common - com.google.api - - - google-auth-library-credentials - com.google.auth - - - google-auth-library-oauth2-http - com.google.auth - - - google-http-client - com.google.http-client - - - google-http-client-gson - com.google.http-client - - - protobuf-java - com.google.protobuf - - - jsr305 - com.google.code.findbugs - - - false - - - com.google.protobuf - protobuf-java-util - 3.19.1 - compile - - - protobuf-java - com.google.protobuf - - - guava - com.google.guava - - - error_prone_annotations - com.google.errorprone - - - j2objc-annotations - com.google.j2objc - - - jsr305 - com.google.code.findbugs - - - gson - com.google.code.gson - - - false - - - com.google.code.gson - gson - 2.8.9 - compile - false - - - com.google.api.grpc - proto-google-common-protos - 2.6.0 - compile - - - protobuf-java - com.google.protobuf - - - false - - - com.google.api.grpc - proto-google-iam-v1 - 1.1.7 - compile - - - protobuf-java - com.google.protobuf - - - proto-google-common-protos - com.google.api.grpc - - - false - - - com.google.auth - google-auth-library-credentials - 1.3.0 - compile - false - - - com.google.http-client - google-http-client-gson - 1.40.1 - compile - - - google-http-client - com.google.http-client - - - gson - com.google.code.gson - - - false - - - com.google.protobuf - protobuf-java - 3.19.1 - compile - false - - - com.google.cloud - google-cloud-core-http - 2.3.0 - compile - - - google-cloud-core - com.google.cloud - - - google-auth-library-credentials - com.google.auth - - - google-auth-library-oauth2-http - com.google.auth - - - google-http-client - com.google.http-client - - - guava - com.google.guava - - - google-api-client - com.google.api-client - - - google-http-client-appengine - com.google.http-client - - - gax - com.google.api - - - gax-httpjson - com.google.api - - - jsr305 - com.google.code.findbugs - - - opencensus-api - io.opencensus - - - opencensus-contrib-http-util - io.opencensus - - - api-common - com.google.api - - - false - - - com.google.api-client - google-api-client - 1.32.2 - compile - - - google-oauth-client - com.google.oauth-client - - - google-http-client-gson - com.google.http-client - - - guava - com.google.guava - - - google-http-client-apache-v2 - com.google.http-client - - - httpcore - org.apache.httpcomponents - - - httpclient - org.apache.httpcomponents - - - google-http-client - com.google.http-client - - - false - - - com.google.oauth-client - google-oauth-client - 1.32.1 - compile - - - google-http-client - com.google.http-client - - - guava - com.google.guava - - - false - - - com.google.http-client - google-http-client-apache-v2 - 1.40.1 - compile - - - google-http-client - com.google.http-client - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - false - - - com.google.http-client - google-http-client-appengine - 1.40.1 - compile - - - google-http-client - com.google.http-client - - - appengine-api-1.0-sdk - com.google.appengine - - - false - - - com.google.api - gax-httpjson - 0.92.0 - compile - - - gax - com.google.api - - - api-common - com.google.api - - - proto-google-common-protos - com.google.api.grpc - - - google-auth-library-credentials - com.google.auth - - - gson - com.google.code.gson - - - guava - com.google.guava - - - google-http-client - com.google.http-client - - - protobuf-java - com.google.protobuf - - - threetenbp - org.threeten - - - google-auth-library-oauth2-http - com.google.auth - - - jsr305 - com.google.code.findbugs - - - google-http-client-gson - com.google.http-client - - - protobuf-java-util - com.google.protobuf - - - false - - - io.opencensus - opencensus-api - 0.28.0 - compile - - - grpc-context - io.grpc - - - false - - - io.grpc - grpc-context - 1.42.1 - compile - false - - - io.opencensus - opencensus-contrib-http-util - 0.28.0 - compile - - - opencensus-api - io.opencensus - - - guava - com.google.guava - - - false - - - com.google.http-client - google-http-client-jackson2 - 1.40.1 - compile - - - google-http-client - com.google.http-client - - - jackson-core - com.fasterxml.jackson.core - - - false - - - com.fasterxml.jackson.core - jackson-core - 2.13.0 - compile - false - - - javax.annotation - javax.annotation-api - 1.3.2 - compile - false - - - com.google.auto.value - auto-value-annotations - 1.8.2 - provided - false - - - com.google.http-client - google-http-client - 1.40.1 - compile - - - httpclient - org.apache.httpcomponents - - - httpcore - org.apache.httpcomponents - - - jsr305 - com.google.code.findbugs - - - guava - com.google.guava - - - j2objc-annotations - com.google.j2objc - - - opencensus-api - io.opencensus - - - opencensus-contrib-http-util - io.opencensus - - - false - - - org.apache.httpcomponents - httpclient - 4.5.13 - compile - - - httpcore - org.apache.httpcomponents - - - commons-logging - commons-logging - - - commons-codec - commons-codec - - - false - - - commons-logging - commons-logging - 1.2 - compile - - - log4j - log4j - - - logkit - logkit - - - avalon-framework - avalon-framework - - - servlet-api - javax.servlet - - - false - - - commons-codec - commons-codec - 1.15 - compile - false - - - org.apache.httpcomponents - httpcore - 4.4.14 - compile - false - - - com.google.j2objc - j2objc-annotations - 1.3 - compile - false - - - org.checkerframework - checker-compat-qual - 2.5.5 - compile - false - - - com.google.auth - google-auth-library-oauth2-http - 1.3.0 - compile - - - auto-value-annotations - com.google.auto.value - - - jsr305 - com.google.code.findbugs - - - google-auth-library-credentials - com.google.auth - - - google-http-client - com.google.http-client - - - google-http-client-gson - com.google.http-client - - - guava - com.google.guava - - - false - - - com.google.code.findbugs - jsr305 - 3.0.2 - compile - false - - - com.google.apis - google-api-services-bigquery - v2-rev20211106-1.32.1 - compile - - - google-api-client - com.google.api-client - - - false - - - com.google.api - api-common - 2.1.0 - compile - - - guava - com.google.guava - - - jsr305 - com.google.code.findbugs - - - javax.annotation-api - javax.annotation - - - auto-value-annotations - com.google.auto.value - - - false - - - com.google.guava - guava - 31.0.1-jre - compile - - - failureaccess - com.google.guava - - - listenablefuture - com.google.guava - - - jsr305 - com.google.code.findbugs - - - checker-qual - org.checkerframework - - - error_prone_annotations - com.google.errorprone - - - j2objc-annotations - com.google.j2objc - - - srczip - jdk - - - false - - - com.google.guava - failureaccess - 1.0.1 - compile - false - - - com.google.guava - listenablefuture - 9999.0-empty-to-avoid-conflict-with-guava - compile - false - - - org.checkerframework - checker-qual - 3.19.0 - compile - false - - - com.google.api - gax - 2.7.0 - compile - - - api-common - com.google.api - - - google-auth-library-credentials - com.google.auth - - - threetenbp - org.threeten - - - google-auth-library-oauth2-http - com.google.auth - - - jsr305 - com.google.code.findbugs - - - guava - com.google.guava - - - opencensus-api - io.opencensus - - - false - - - org.threeten - threetenbp - 1.5.2 - compile - false - - - com.google.errorprone - error_prone_annotations - 2.10.0 - compile - false - - - From 76cd0926b4296ee9e1d95917622e0871f6cc6830 Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Wed, 8 Dec 2021 15:03:37 -0500 Subject: [PATCH 4/8] refactoring, removing unnecesary variables, adding comments --- .../google/cloud/bigquery/BigQueryImpl.java | 9 +++---- .../cloud/bigquery/BigQueryImplTest.java | 26 +++++++++++++++++++ 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index 8a5619898..ffc038597 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -357,7 +357,6 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { // This makes it difficult to translate without throwing. // Fixing this entails some work on BaseServiceException.translate. // Since that affects a bunch of APIs, we should fix this as a separate change. - final JobInfo jobInfoCopy = jobInfo; final JobId[] finalJobId = new JobId[1]; try { try { @@ -368,11 +367,11 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { @Override public com.google.api.services.bigquery.model.Job call() { if(idRandom){ - // generate new random job - JobInfo newJobInfo = jobInfoCopy.toBuilder().setJobId(idProvider.get()).build(); + // re-generate a new random job with the same jobInfo when jobId is not provided by the user + JobInfo recreatedJobInfo = jobInfo.toBuilder().setJobId(idProvider.get()).build(); com.google.api.services.bigquery.model.Job newJobPb = - newJobInfo.setProjectId(getOptions().getProjectId()).toPb(); - finalJobId[0] = newJobInfo.getJobId(); + recreatedJobInfo.setProjectId(getOptions().getProjectId()).toPb(); + finalJobId[0] = recreatedJobInfo.getJobId(); return bigQueryRpc.create(newJobPb, optionsMap); } else { diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index 760c84f32..ae8abd9a4 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -1529,6 +1529,32 @@ public void testCreateJobSuccess() { verify(bigqueryRpcMock).create(jobCapture.capture(), eq(EMPTY_RPC_OPTIONS)); } + @Test + public void testCreateJobFailureShouldRetry() { + when(bigqueryRpcMock.create(jobCapture.capture(), eq(EMPTY_RPC_OPTIONS))) + .thenThrow(new BigQueryException(500, "InternalError")) + .thenThrow(new BigQueryException(502, "Bad Gateway")) + .thenThrow(new BigQueryException(503, "Service Unavailable")) + .thenThrow( + new BigQueryException( + 400, RATE_LIMIT_ERROR_MSG)) // retrial on based on RATE_LIMIT_EXCEEDED_MSG + .thenThrow( + new BigQueryException( + 200, RATE_LIMIT_ERROR_MSG)) + .thenReturn(newJobPb()); + + bigquery = options.getService(); + bigquery = + options + .toBuilder() + .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) + .build() + .getService(); + + ((BigQueryImpl) bigquery).create(JobInfo.of(QUERY_JOB_CONFIGURATION_FOR_DMLQUERY)); + verify(bigqueryRpcMock, times(6)).create(jobCapture.capture(), eq(EMPTY_RPC_OPTIONS)); + } + @Test public void testCreateJobWithSelectedFields() { when(bigqueryRpcMock.create( From 84ae60f079a8c206512c8b7122852251722013e1 Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Wed, 8 Dec 2021 15:14:09 -0500 Subject: [PATCH 5/8] linting code --- .../google/cloud/bigquery/BigQueryImpl.java | 15 +++++----- .../bigquery/BigQueryRetryAlgorithm.java | 13 +++++---- .../cloud/bigquery/BigQueryImplTest.java | 28 +++++++++---------- 3 files changed, 29 insertions(+), 27 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java index ffc038597..16814e99a 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryImpl.java @@ -366,17 +366,18 @@ Job create(JobInfo jobInfo, Supplier idProvider, JobOption... options) { new Callable() { @Override public com.google.api.services.bigquery.model.Job call() { - if(idRandom){ - // re-generate a new random job with the same jobInfo when jobId is not provided by the user - JobInfo recreatedJobInfo = jobInfo.toBuilder().setJobId(idProvider.get()).build(); + if (idRandom) { + // re-generate a new random job with the same jobInfo when jobId is not + // provided by the user + JobInfo recreatedJobInfo = + jobInfo.toBuilder().setJobId(idProvider.get()).build(); com.google.api.services.bigquery.model.Job newJobPb = - recreatedJobInfo.setProjectId(getOptions().getProjectId()).toPb(); + recreatedJobInfo.setProjectId(getOptions().getProjectId()).toPb(); finalJobId[0] = recreatedJobInfo.getJobId(); return bigQueryRpc.create(newJobPb, optionsMap); - } - else { + } else { com.google.api.services.bigquery.model.Job jobPb = - jobInfo.setProjectId(getOptions().getProjectId()).toPb(); + jobInfo.setProjectId(getOptions().getProjectId()).toPb(); return bigQueryRpc.create(jobPb, optionsMap); } } diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java index bba0aaecb..ba0e4727e 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java @@ -72,7 +72,8 @@ public boolean shouldRetry( // the exception messages boolean shouldRetry = (shouldRetryBasedOnResult(context, previousThrowable, previousResponse) - || shouldRetryBasedOnBigQueryRetryConfig(previousThrowable, bigQueryRetryConfig, previousResponse)) + || shouldRetryBasedOnBigQueryRetryConfig( + previousThrowable, bigQueryRetryConfig, previousResponse)) && shouldRetryBasedOnTiming(context, nextAttemptSettings); if (LOG.isLoggable(Level.FINEST)) { @@ -92,15 +93,17 @@ public boolean shouldRetry( } private boolean shouldRetryBasedOnBigQueryRetryConfig( - Throwable previousThrowable, BigQueryRetryConfig bigQueryRetryConfig, ResponseT previousResponse) { + Throwable previousThrowable, + BigQueryRetryConfig bigQueryRetryConfig, + ResponseT previousResponse) { /* We are deciding if a given error should be retried on the basis of error message. Cannot rely on Error/Status code as for example error code 400 (which is not retriable) could be thrown due to rateLimitExceed, which is retriable */ String errorDesc = null; - if (previousThrowable != null ){ + if (previousThrowable != null) { errorDesc = previousThrowable.getMessage(); - } else if (previousResponse != null){ + } else if (previousResponse != null) { /* In some cases error messages may come without an exception e.g. status code 200 with a rate limit exceeded for job create @@ -109,7 +112,7 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( errorDesc = previousResponse.toString(); } - if (errorDesc != null){ + if (errorDesc != null) { errorDesc = errorDesc.toLowerCase(); // for case insensitive comparison for (Iterator retriableMessages = bigQueryRetryConfig.getRetriableErrorMessages().iterator(); diff --git a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java index ae8abd9a4..b00d4b860 100644 --- a/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java +++ b/google-cloud-bigquery/src/test/java/com/google/cloud/bigquery/BigQueryImplTest.java @@ -1532,24 +1532,22 @@ public void testCreateJobSuccess() { @Test public void testCreateJobFailureShouldRetry() { when(bigqueryRpcMock.create(jobCapture.capture(), eq(EMPTY_RPC_OPTIONS))) - .thenThrow(new BigQueryException(500, "InternalError")) - .thenThrow(new BigQueryException(502, "Bad Gateway")) - .thenThrow(new BigQueryException(503, "Service Unavailable")) - .thenThrow( - new BigQueryException( - 400, RATE_LIMIT_ERROR_MSG)) // retrial on based on RATE_LIMIT_EXCEEDED_MSG - .thenThrow( - new BigQueryException( - 200, RATE_LIMIT_ERROR_MSG)) - .thenReturn(newJobPb()); + .thenThrow(new BigQueryException(500, "InternalError")) + .thenThrow(new BigQueryException(502, "Bad Gateway")) + .thenThrow(new BigQueryException(503, "Service Unavailable")) + .thenThrow( + new BigQueryException( + 400, RATE_LIMIT_ERROR_MSG)) // retrial on based on RATE_LIMIT_EXCEEDED_MSG + .thenThrow(new BigQueryException(200, RATE_LIMIT_ERROR_MSG)) + .thenReturn(newJobPb()); bigquery = options.getService(); bigquery = - options - .toBuilder() - .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) - .build() - .getService(); + options + .toBuilder() + .setRetrySettings(ServiceOptions.getDefaultRetrySettings()) + .build() + .getService(); ((BigQueryImpl) bigquery).create(JobInfo.of(QUERY_JOB_CONFIGURATION_FOR_DMLQUERY)); verify(bigqueryRpcMock, times(6)).create(jobCapture.capture(), eq(EMPTY_RPC_OPTIONS)); From 2482f78562e44bac348230370e8e879385684a79 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 14 Dec 2021 19:04:45 +0000 Subject: [PATCH 6/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5be608437..7ecd9c640 100644 --- a/README.md +++ b/README.md @@ -51,20 +51,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:24.0.0') +implementation platform('com.google.cloud:libraries-bom:24.1.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.5.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.5.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.5.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.5.1" ``` ## Authentication From 514b57a2aba0416ab63b1bf6090414577a6543e4 Mon Sep 17 00:00:00 2001 From: Owl Bot Date: Tue, 14 Dec 2021 19:05:04 +0000 Subject: [PATCH 7/8] =?UTF-8?q?=F0=9F=A6=89=20Updates=20from=20OwlBot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5be608437..7ecd9c640 100644 --- a/README.md +++ b/README.md @@ -51,20 +51,20 @@ If you are using Maven without BOM, add this to your dependencies: If you are using Gradle 5.x or later, add this to your dependencies ```Groovy -implementation platform('com.google.cloud:libraries-bom:24.0.0') +implementation platform('com.google.cloud:libraries-bom:24.1.0') implementation 'com.google.cloud:google-cloud-bigquery' ``` If you are using Gradle without BOM, add this to your dependencies ```Groovy -implementation 'com.google.cloud:google-cloud-bigquery:2.5.0' +implementation 'com.google.cloud:google-cloud-bigquery:2.5.1' ``` If you are using SBT, add this to your dependencies ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.5.0" +libraryDependencies += "com.google.cloud" % "google-cloud-bigquery" % "2.5.1" ``` ## Authentication From ca2931c54b106af69499254347c9292807787d69 Mon Sep 17 00:00:00 2001 From: Franklin Whaite Date: Tue, 14 Dec 2021 14:09:45 -0500 Subject: [PATCH 8/8] refactoring, removing comments --- .../bigquery/BigQueryRetryAlgorithm.java | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java index ba0e4727e..af472430f 100644 --- a/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java +++ b/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/BigQueryRetryAlgorithm.java @@ -134,31 +134,6 @@ private boolean shouldRetryBasedOnBigQueryRetryConfig( } } } - // status code 200 - /* - if (previousResponse != null && (errorDesc = previousResponse.toString()) != null) { - errorDesc = errorDesc.toLowerCase(); // for case insensitive comparison - for (Iterator retriableMessages = - bigQueryRetryConfig.getRetriableErrorMessages().iterator(); - retriableMessages.hasNext(); ) { - if (errorDesc.contains( - retriableMessages - .next() - .toLowerCase())) { // Error message should be retried, implementing cases - // insensitive match - return true; - } - } - - // Check if there's a regex which matches the error message. This avoids too many regex - // matches which is expensive - for (Iterator retriableRegExes = bigQueryRetryConfig.getRetriableRegExes().iterator(); - retriableRegExes.hasNext(); ) { - if (matchRegEx(retriableRegExes.next(), errorDesc)) { - return true; - } - } - }*/ return false; }