BigQuery: Allow job to extract or load table across projects (#4183)

* Fix #3924 ExtractJobConfiguration's setProjectId makes cross-project BQ extracts impossible
googleapis · Dec 6, 2018 · d122744 · d122744
1 parent f3d0703
commit d122744
Show file tree

Hide file tree

Showing 8 changed files with 95 additions and 15 deletions.
diff --git a/...s/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java b/...s/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/CopyJobConfiguration.java
@@ -21,6 +21,7 @@
 import com.google.api.services.bigquery.model.JobConfigurationTableCopy;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.util.List;
@@ -216,10 +217,15 @@ CopyJobConfiguration setProjectId(final String projectId) {
             new Function<TableId, TableId>() {
               @Override
               public TableId apply(TableId tableId) {
-                return tableId.setProjectId(projectId);
+                if (Strings.isNullOrEmpty(tableId.getProject())) {
+                  return tableId.setProjectId(projectId);
+                }
+                return tableId;
               }
             }));
-    builder.setDestinationTable(getDestinationTable().setProjectId(projectId));
+    if (Strings.isNullOrEmpty(getDestinationTable().getProject())) {
+      builder.setDestinationTable(getDestinationTable().setProjectId(projectId));
+    }
     return builder.build();
   }
 

diff --git a/...oogle-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java b/...oogle-cloud-bigquery/src/main/java/com/google/cloud/bigquery/ExtractJobConfiguration.java
@@ -20,6 +20,7 @@
 
 import com.google.api.services.bigquery.model.JobConfigurationExtract;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import java.util.Objects;
@@ -215,7 +216,10 @@ public int hashCode() {
 
   @Override
   ExtractJobConfiguration setProjectId(String projectId) {
-    return toBuilder().setSourceTable(getSourceTable().setProjectId(projectId)).build();
+    if (Strings.isNullOrEmpty(getSourceTable().getProject())) {
+      return toBuilder().setSourceTable(getSourceTable().setProjectId(projectId)).build();
+    }
+    return this;
   }
 
   @Override

diff --git a/...s/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java b/...s/google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/LoadJobConfiguration.java
@@ -20,6 +20,7 @@
 
 import com.google.api.services.bigquery.model.JobConfigurationLoad;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.primitives.Ints;
 import java.util.List;
@@ -398,7 +399,10 @@ public int hashCode() {
 
   @Override
   LoadJobConfiguration setProjectId(String projectId) {
-    return toBuilder().setDestinationTable(getDestinationTable().setProjectId(projectId)).build();
+    if (Strings.isNullOrEmpty(getDestinationTable().getProject())) {
+      return toBuilder().setDestinationTable(getDestinationTable().setProjectId(projectId)).build();
+    }
+    return this;
   }
 
   @Override

diff --git a/.../google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java b/.../google-cloud-bigquery/src/main/java/com/google/cloud/bigquery/QueryJobConfiguration.java
@@ -26,6 +26,7 @@
 import com.google.cloud.bigquery.JobInfo.WriteDisposition;
 import com.google.common.base.Function;
 import com.google.common.base.MoreObjects.ToStringHelper;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Iterables;
@@ -769,7 +770,8 @@ public int hashCode() {
   @Override
   QueryJobConfiguration setProjectId(String projectId) {
     Builder builder = toBuilder();
-    if (getDestinationTable() != null) {
+    if (getDestinationTable() != null
+        && Strings.isNullOrEmpty(getDestinationTable().getProject())) {
       builder.setDestinationTable(getDestinationTable().setProjectId(projectId));
     }
     if (getDefaultDataset() != null) {

diff --git a/...ogle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java b/...ogle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/CopyJobConfigurationTest.java
@@ -22,12 +22,15 @@
 
 import com.google.cloud.bigquery.JobInfo.CreateDisposition;
 import com.google.cloud.bigquery.JobInfo.WriteDisposition;
+import com.google.common.base.Function;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import java.util.List;
 import org.junit.Test;
 
 public class CopyJobConfigurationTest {
 
+  private static final String TEST_PROJECT_ID = "test-project-id";
   private static final TableId SOURCE_TABLE = TableId.of("dataset", "sourceTable");
   private static final List<TableId> SOURCE_TABLES =
       ImmutableList.of(
@@ -115,10 +118,34 @@ public void testToPbAndFromPb() {
 
   @Test
   public void testSetProjectId() {
-    CopyJobConfiguration configuration = COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId("p");
-    assertEquals("p", configuration.getDestinationTable().getProject());
+    CopyJobConfiguration configuration =
+        COPY_JOB_CONFIGURATION_MULTIPLE_TABLES.setProjectId(TEST_PROJECT_ID);
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
     for (TableId sourceTable : configuration.getSourceTables()) {
-      assertEquals("p", sourceTable.getProject());
+      assertEquals(TEST_PROJECT_ID, sourceTable.getProject());
+    }
+  }
+
+  @Test
+  public void testSetProjectIdDoNotOverride() {
+    CopyJobConfiguration configuration =
+        COPY_JOB_CONFIGURATION_MULTIPLE_TABLES
+            .toBuilder()
+            .setSourceTables(
+                Lists.transform(
+                    SOURCE_TABLES,
+                    new Function<TableId, TableId>() {
+                      @Override
+                      public TableId apply(TableId tableId) {
+                        return tableId.setProjectId(TEST_PROJECT_ID);
+                      }
+                    }))
+            .setDestinationTable(DESTINATION_TABLE.setProjectId(TEST_PROJECT_ID))
+            .build()
+            .setProjectId("do-not-update");
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
+    for (TableId sourceTable : configuration.getSourceTables()) {
+      assertEquals(TEST_PROJECT_ID, sourceTable.getProject());
     }
   }
 

diff --git a/...e-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java b/...e-cloud-bigquery/src/test/java/com/google/cloud/bigquery/ExtractJobConfigurationTest.java
@@ -26,6 +26,7 @@
 
 public class ExtractJobConfigurationTest {
 
+  private static final String TEST_PROJECT_ID = "test-project-id";
   private static final List<String> DESTINATION_URIS = ImmutableList.of("uri1", "uri2");
   private static final String DESTINATION_URI = "uri1";
   private static final TableId TABLE_ID = TableId.of("dataset", "table");
@@ -118,8 +119,19 @@ public void testToPbAndFromPb() {
 
   @Test
   public void testSetProjectId() {
-    ExtractJobConfiguration configuration = EXTRACT_CONFIGURATION.setProjectId("p");
-    assertEquals("p", configuration.getSourceTable().getProject());
+    ExtractJobConfiguration configuration = EXTRACT_CONFIGURATION.setProjectId(TEST_PROJECT_ID);
+    assertEquals(TEST_PROJECT_ID, configuration.getSourceTable().getProject());
+  }
+
+  @Test
+  public void testSetProjectIdDoNotOverride() {
+    ExtractJobConfiguration configuration =
+        EXTRACT_CONFIGURATION
+            .toBuilder()
+            .setSourceTable(TABLE_ID.setProjectId(TEST_PROJECT_ID))
+            .build()
+            .setProjectId("do-not-update");
+    assertEquals(TEST_PROJECT_ID, configuration.getSourceTable().getProject());
   }
 
   @Test

diff --git a/...ogle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java b/...ogle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/LoadJobConfigurationTest.java
@@ -29,6 +29,7 @@
 
 public class LoadJobConfigurationTest {
 
+  private static final String TEST_PROJECT_ID = "test-project-id";
   private static final CsvOptions CSV_OPTIONS =
       CsvOptions.newBuilder()
           .setAllowJaggedRows(true)
@@ -146,8 +147,19 @@ public void testToPbAndFromPb() {
 
   @Test
   public void testSetProjectId() {
-    LoadConfiguration configuration = LOAD_CONFIGURATION_CSV.setProjectId("p");
-    assertEquals("p", configuration.getDestinationTable().getProject());
+    LoadConfiguration configuration = LOAD_CONFIGURATION_CSV.setProjectId(TEST_PROJECT_ID);
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
+  }
+
+  @Test
+  public void testSetProjectIdDoNotOverride() {
+    LoadConfiguration configuration =
+        LOAD_CONFIGURATION_CSV
+            .toBuilder()
+            .setDestinationTable(TABLE_ID.setProjectId(TEST_PROJECT_ID))
+            .build()
+            .setProjectId("do-not-update");
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
   }
 
   @Test

diff --git a/...gle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java b/...gle-cloud-bigquery/src/test/java/com/google/cloud/bigquery/QueryJobConfigurationTest.java
@@ -33,6 +33,7 @@
 
 public class QueryJobConfigurationTest {
 
+  private static final String TEST_PROJECT_ID = "test-project-id";
   private static final String QUERY = "BigQuery SQL";
   private static final DatasetId DATASET_ID = DatasetId.of("dataset");
   private static final TableId TABLE_ID = TableId.of("dataset", "table");
@@ -140,9 +141,21 @@ public void testToPbAndFromPb() {
 
   @Test
   public void testSetProjectId() {
-    QueryJobConfiguration configuration = QUERY_JOB_CONFIGURATION.setProjectId("p");
-    assertEquals("p", configuration.getDefaultDataset().getProject());
-    assertEquals("p", configuration.getDestinationTable().getProject());
+    QueryJobConfiguration configuration = QUERY_JOB_CONFIGURATION.setProjectId(TEST_PROJECT_ID);
+    assertEquals(TEST_PROJECT_ID, configuration.getDefaultDataset().getProject());
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
+  }
+
+  @Test
+  public void testSetProjectIdDoNotOverride() {
+    QueryJobConfiguration configuration =
+        QUERY_JOB_CONFIGURATION
+            .toBuilder()
+            .setDestinationTable(TABLE_ID.setProjectId(TEST_PROJECT_ID))
+            .build()
+            .setProjectId("update-only-on-dataset");
+    assertEquals("update-only-on-dataset", configuration.getDefaultDataset().getProject());
+    assertEquals(TEST_PROJECT_ID, configuration.getDestinationTable().getProject());
   }
 
   @Test