From b7ebd73754fe51a4835f676e429b95b80f8114a5 Mon Sep 17 00:00:00 2001 From: Phong Chuong <147636638+PhongChuong@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:27:38 -0500 Subject: [PATCH] Fix: Fix issue where Universe Domain is not correctly set. (#2423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix: Fix issue where Universe Domain is not correctly set. This PR fixes an issue where the universe domain is not correctly set. This also adds integration tests to verify the change. * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Add universe domain integration tests for v1beta1 * Fix issue with universe domain for v1beta and add functional integration test for universe domain. * Fix integration test failure due to commented out code. Also close newly created clients. --------- Co-authored-by: Owl Bot --- .../v1/stub/EnhancedBigQueryReadStub.java | 1 + .../EnhancedBigQueryReadStubSettings.java | 1 - .../stub/EnhancedBigQueryStorageStub.java | 2 +- .../EnhancedBigQueryStorageStubSettings.java | 1 - .../stub/EnhancedBigQueryReadStub.java | 1 + .../EnhancedBigQueryReadStubSettings.java | 1 - .../storage/v1/it/ITBigQueryStorageTest.java | 219 ++++++++++++++++- .../v1beta1/it/ITBigQueryStorageTest.java | 226 ++++++++++++++++++ .../v1beta2/it/ITBigQueryStorageTest.java | 219 +++++++++++++++++ 9 files changed, 666 insertions(+), 5 deletions(-) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStub.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStub.java index 1d97541611..d60ea02596 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStub.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStub.java @@ -70,6 +70,7 @@ public static EnhancedBigQueryReadStub create( // Configure the base settings. BigQueryReadStubSettings.Builder baseSettingsBuilder = BigQueryReadStubSettings.newBuilder() + .setUniverseDomain(settings.getUniverseDomain()) .setTransportChannelProvider(settings.getTransportChannelProvider()) .setEndpoint(settings.getEndpoint()) .setHeaderProvider(settings.getHeaderProvider()) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStubSettings.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStubSettings.java index 190f355779..20d5074699 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStubSettings.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/stub/EnhancedBigQueryReadStubSettings.java @@ -159,7 +159,6 @@ protected Builder(ClientContext clientContext) { // Defaults provider BigQueryReadStubSettings.Builder baseDefaults = BigQueryReadStubSettings.newBuilder(); - setEndpoint(baseDefaults.getEndpoint()); setTransportChannelProvider(defaultTransportChannelProvider()); setCredentialsProvider(baseDefaults.getCredentialsProvider()); setStreamWatchdogCheckInterval(baseDefaults.getStreamWatchdogCheckInterval()); diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStub.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStub.java index 5a1940dff7..e729c2c124 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStub.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStub.java @@ -74,8 +74,8 @@ public static EnhancedBigQueryStorageStub create( // Configure the base settings. BigQueryStorageStubSettings.Builder baseSettingsBuilder = BigQueryStorageStubSettings.newBuilder() + .setUniverseDomain(settings.getUniverseDomain()) .setTransportChannelProvider(settings.getTransportChannelProvider()) - .setEndpoint(settings.getEndpoint()) .setHeaderProvider(settings.getHeaderProvider()) .setCredentialsProvider(settings.getCredentialsProvider()) .setStreamWatchdogCheckInterval(settings.getStreamWatchdogCheckInterval()) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStubSettings.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStubSettings.java index 1b77bb5aec..c60ac88dce 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStubSettings.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta1/stub/EnhancedBigQueryStorageStubSettings.java @@ -186,7 +186,6 @@ protected Builder(ClientContext clientContext) { // Defaults provider BigQueryStorageStubSettings.Builder baseDefaults = BigQueryStorageStubSettings.newBuilder(); - setEndpoint(baseDefaults.getEndpoint()); setTransportChannelProvider(defaultTransportChannelProvider()); setCredentialsProvider(baseDefaults.getCredentialsProvider()); setStreamWatchdogCheckInterval(baseDefaults.getStreamWatchdogCheckInterval()); diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStub.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStub.java index 28870b7a47..0723febba8 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStub.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStub.java @@ -70,6 +70,7 @@ public static EnhancedBigQueryReadStub create( // Configure the base settings. BigQueryReadStubSettings.Builder baseSettingsBuilder = BigQueryReadStubSettings.newBuilder() + .setUniverseDomain(settings.getUniverseDomain()) .setTransportChannelProvider(settings.getTransportChannelProvider()) .setEndpoint(settings.getEndpoint()) .setHeaderProvider(settings.getHeaderProvider()) diff --git a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStubSettings.java b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStubSettings.java index 6f6c871bcf..f6ef48b986 100644 --- a/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStubSettings.java +++ b/google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1beta2/stub/EnhancedBigQueryReadStubSettings.java @@ -159,7 +159,6 @@ protected Builder(ClientContext clientContext) { // Defaults provider BigQueryReadStubSettings.Builder baseDefaults = BigQueryReadStubSettings.newBuilder(); - setEndpoint(baseDefaults.getEndpoint()); setTransportChannelProvider(defaultTransportChannelProvider()); setCredentialsProvider(baseDefaults.getCredentialsProvider()); setStreamWatchdogCheckInterval(baseDefaults.getStreamWatchdogCheckInterval()); diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageTest.java index 4e5f0ee127..e18cb1f54b 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/it/ITBigQueryStorageTest.java @@ -16,15 +16,20 @@ package com.google.cloud.bigquery.storage.v1.it; +import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.core.InstantiatingExecutorProvider; import com.google.api.gax.rpc.ServerStream; +import com.google.api.gax.rpc.UnauthenticatedException; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.RetryOption; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; @@ -54,7 +59,9 @@ import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.common.base.Preconditions; import com.google.protobuf.Timestamp; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -82,7 +89,6 @@ /** Integration tests for BigQuery Storage API. */ public class ITBigQueryStorageTest { - private static final Logger LOG = Logger.getLogger(ITBigQueryStorageTest.class.getName()); private static final String DATASET = RemoteBigQueryHelper.generateDatasetName(); private static final String DESCRIPTION = "BigQuery Storage Java client test dataset"; @@ -91,6 +97,66 @@ public class ITBigQueryStorageTest { private static String parentProjectId; private static BigQuery bigquery; + private static final String FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"googleapis.com\"\n" + + "}"; + + private static final String FAKE_JSON_CRED_WITH_INVALID_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"fake.domain\"\n" + + "}"; + @BeforeClass public static void beforeClass() throws IOException { client = BigQueryReadClient.create(); @@ -859,6 +925,147 @@ public void testSimpleReadWithBackgroundExecutorProvider() throws IOException { assertEquals(164_656, rowCount); } + @Test + public void testUniverseDomainWithInvalidUniverseDomain() throws IOException { + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create(loadCredentials(FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + try { + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testInvalidUniverseDomainWithMismatchCredentials() throws IOException { + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create( + loadCredentials(FAKE_JSON_CRED_WITH_INVALID_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + try { + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testUniverseDomainWithMatchingDomain() throws IOException { + // Test a valid domain using the default credentials and Google default universe domain. + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder().setUniverseDomain("googleapis.com").build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build(); + + long rowCount = 0; + ServerStream stream = client.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(164_656, rowCount); + localClient.close(); + } + + public void testUniverseDomain() throws IOException { + // This test is not yet part presubmit integration test as it requires the apis-tpclp.goog + // universe domain credentials. + // Test a valid read session in the universe domain gdutst. + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder().setUniverseDomain("apis-tpclp.goog").build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "google-tpc-testing-environment:cloudsdk-test-project", + /* datasetId = */ "tpc_demo_dataset", + /* tableId = */ "new_table"); + + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build(); + + long rowCount = 0; + ServerStream stream = localClient.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(1, rowCount); + localClient.close(); + } + /** * Reads to the specified row offset within the stream. If the stream does not have the desired * rows to read, it will read all of them. @@ -1015,4 +1222,14 @@ private Job RunQueryJobAndExpectSuccess(QueryJobConfiguration configuration) return completedJob; } + + static GoogleCredentials loadCredentials(String credentialFile) { + try { + InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes()); + return GoogleCredentials.fromStream(keyStream); + } catch (IOException e) { + fail("Couldn't create fake JSON credentials."); + } + return null; + } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta1/it/ITBigQueryStorageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta1/it/ITBigQueryStorageTest.java index 79727cd1f2..933bb77ab1 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta1/it/ITBigQueryStorageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta1/it/ITBigQueryStorageTest.java @@ -16,13 +16,18 @@ package com.google.cloud.bigquery.storage.v1beta1.it; +import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.rpc.ServerStream; +import com.google.api.gax.rpc.UnauthenticatedException; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.RetryOption; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; @@ -39,6 +44,7 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TimePartitioning; import com.google.cloud.bigquery.storage.v1beta1.BigQueryStorageClient; +import com.google.cloud.bigquery.storage.v1beta1.BigQueryStorageSettings; import com.google.cloud.bigquery.storage.v1beta1.ReadOptions.TableReadOptions; import com.google.cloud.bigquery.storage.v1beta1.Storage.CreateReadSessionRequest; import com.google.cloud.bigquery.storage.v1beta1.Storage.DataFormat; @@ -54,7 +60,9 @@ import com.google.common.base.Preconditions; import com.google.protobuf.TextFormat; import com.google.protobuf.Timestamp; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -91,6 +99,66 @@ public class ITBigQueryStorageTest { private static String parentProjectId; private static BigQuery bigquery; + private static final String FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"googleapis.com\"\n" + + "}"; + + private static final String FAKE_JSON_CRED_WITH_INVALID_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"fake.domain\"\n" + + "}"; + @BeforeClass public static void beforeClass() throws IOException { client = BigQueryStorageClient.create(); @@ -825,6 +893,154 @@ public void testStructAndArraySqlTypes() throws InterruptedException, IOExceptio assertEquals(rowAssertMessage, new Utf8("abc"), structRecord.get("str_field")); } + @Test + public void testUniverseDomainWithInvalidUniverseDomain() throws IOException { + BigQueryStorageSettings bigQueryStorageSettings = + BigQueryStorageSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create( + loadCredentials(FAKE_JSON_CRED_WITH_INVALID_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + + BigQueryStorageClient localClient = BigQueryStorageClient.create(bigQueryStorageSettings); + + TableReference tableReference = + TableReference.newBuilder() + .setProjectId("bigquery-public-data") + .setDatasetId("samples") + .setTableId("shakespeare") + .build(); + + try { + localClient.createReadSession( + /* tableReference = */ tableReference, + /* parent = */ parentProjectId, + /* requestedStreams = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testInvalidUniverseDomainWithMismatchCredentials() throws IOException { + BigQueryStorageSettings bigQueryStorageSettings = + BigQueryStorageSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create( + loadCredentials(FAKE_JSON_CRED_WITH_INVALID_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + + BigQueryStorageClient localClient = BigQueryStorageClient.create(bigQueryStorageSettings); + + TableReference tableReference = + TableReference.newBuilder() + .setProjectId("bigquery-public-data") + .setDatasetId("samples") + .setTableId("shakespeare") + .build(); + + try { + localClient.createReadSession( + /* tableReference = */ tableReference, + /* parent = */ parentProjectId, + /* requestedStreams = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testUniverseDomainWithMatchingDomain() throws IOException { + // Test a valid domain using the default credentials and Google default universe domain. + BigQueryStorageSettings bigQueryStorageSettings = + BigQueryStorageSettings.newBuilder().setUniverseDomain("googleapis.com").build(); + BigQueryStorageClient localClient = BigQueryStorageClient.create(bigQueryStorageSettings); + + TableReference tableReference = + TableReference.newBuilder() + .setProjectId("bigquery-public-data") + .setDatasetId("samples") + .setTableId("shakespeare") + .build(); + + ReadSession session = + localClient.createReadSession( + /* tableReference = */ tableReference, + /* parent = */ parentProjectId, + /* requestedStreams = */ 1); + + assertEquals( + String.format( + "Did not receive expected number of streams for table reference '%s' CreateReadSession response:%n%s", + TextFormat.shortDebugString(tableReference), session.toString()), + 1, + session.getStreamsCount()); + + StreamPosition readPosition = + StreamPosition.newBuilder().setStream(session.getStreams(0)).build(); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadPosition(readPosition).build(); + + long rowCount = 0; + ServerStream stream = client.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(164_656, rowCount); + localClient.close(); + } + + public void testUniverseDomain() throws IOException { + // This test is not yet part presubmit integration test as it requires the apis-tpclp.goog + // universe domain credentials. + // Test a valid read session in the universe domain gdutst. + BigQueryStorageSettings bigQueryStorageSettings = + BigQueryStorageSettings.newBuilder().setUniverseDomain("apis-tpclp.goog").build(); + BigQueryStorageClient localClient = BigQueryStorageClient.create(bigQueryStorageSettings); + + TableReference tableReference = + TableReference.newBuilder() + .setProjectId("google-tpc-testing-environment:cloudsdk-test-project") + .setDatasetId("tpc_demo_dataset") + .setTableId("new_table") + .build(); + + ReadSession session = + localClient.createReadSession( + /* tableReference = */ tableReference, + /* parent = */ parentProjectId, + /* requestedStreams = */ 1); + + StreamPosition readPosition = + StreamPosition.newBuilder().setStream(session.getStreams(0)).build(); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadPosition(readPosition).build(); + + long rowCount = 0; + ServerStream stream = localClient.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(1, rowCount); + localClient.close(); + } + /** * Reads to the specified row offset within the stream. If the stream does not have the desired * rows to read, it will read all of them. @@ -984,4 +1200,14 @@ private Job RunQueryJobAndExpectSuccess(QueryJobConfiguration configuration) return completedJob; } + + static GoogleCredentials loadCredentials(String credentialFile) { + try { + InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes()); + return GoogleCredentials.fromStream(keyStream); + } catch (IOException e) { + fail("Couldn't create fake JSON credentials."); + } + return null; + } } diff --git a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/it/ITBigQueryStorageTest.java b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/it/ITBigQueryStorageTest.java index 2e0ee030a1..fdf440e44a 100644 --- a/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/it/ITBigQueryStorageTest.java +++ b/google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1beta2/it/ITBigQueryStorageTest.java @@ -16,13 +16,18 @@ package com.google.cloud.bigquery.storage.v1beta2.it; +import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.fail; +import com.google.api.gax.core.FixedCredentialsProvider; import com.google.api.gax.rpc.ServerStream; +import com.google.api.gax.rpc.UnauthenticatedException; +import com.google.auth.oauth2.GoogleCredentials; import com.google.cloud.RetryOption; import com.google.cloud.ServiceOptions; import com.google.cloud.bigquery.BigQuery; @@ -39,6 +44,7 @@ import com.google.cloud.bigquery.TableInfo; import com.google.cloud.bigquery.TimePartitioning; import com.google.cloud.bigquery.storage.v1beta2.BigQueryReadClient; +import com.google.cloud.bigquery.storage.v1beta2.BigQueryReadSettings; import com.google.cloud.bigquery.storage.v1beta2.CreateReadSessionRequest; import com.google.cloud.bigquery.storage.v1beta2.DataFormat; import com.google.cloud.bigquery.storage.v1beta2.ReadRowsRequest; @@ -51,7 +57,9 @@ import com.google.cloud.bigquery.testing.RemoteBigQueryHelper; import com.google.common.base.Preconditions; import com.google.protobuf.Timestamp; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -88,6 +96,66 @@ public class ITBigQueryStorageTest { private static String parentProjectId; private static BigQuery bigquery; + private static final String FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"googleapis.com\"\n" + + "}"; + + private static final String FAKE_JSON_CRED_WITH_INVALID_DOMAIN = + "{\n" + + " \"private_key_id\": \"somekeyid\",\n" + + " \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggS" + + "kAgEAAoIBAQC+K2hSuFpAdrJI\\nnCgcDz2M7t7bjdlsadsasad+fvRSW6TjNQZ3p5LLQY1kSZRqBqylRkzteMOyHg" + + "aR\\n0Pmxh3ILCND5men43j3h4eDbrhQBuxfEMalkG92sL+PNQSETY2tnvXryOvmBRwa/\\nQP/9dJfIkIDJ9Fw9N4" + + "Bhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nknddadwkwewcVxHFhcZJO+XWf6ofLUXpRwiTZakGMn8EE1uVa2" + + "LgczOjwWHGi99MFjxSer5m9\\n1tCa3/KEGKiS/YL71JvjwX3mb+cewlkcmweBKZHM2JPTk0ZednFSpVZMtycjkbLa" + + "\\ndYOS8V85AgMBewECggEBAKksaldajfDZDV6nGqbFjMiizAKJolr/M3OQw16K6o3/\\n0S31xIe3sSlgW0+UbYlF" + + "4U8KifhManD1apVSC3csafaspP4RZUHFhtBywLO9pR5c\\nr6S5aLp+gPWFyIp1pfXbWGvc5VY/v9x7ya1VEa6rXvL" + + "sKupSeWAW4tMj3eo/64ge\\nsdaceaLYw52KeBYiT6+vpsnYrEkAHO1fF/LavbLLOFJmFTMxmsNaG0tuiJHgjshB\\" + + "n82DpMCbXG9YcCgI/DbzuIjsdj2JC1cascSP//3PmefWysucBQe7Jryb6NQtASmnv\\nCdDw/0jmZTEjpe4S1lxfHp" + + "lAhHFtdgYTvyYtaLZiVVkCgYEA8eVpof2rceecw/I6\\n5ng1q3Hl2usdWV/4mZMvR0fOemacLLfocX6IYxT1zA1FF" + + "JlbXSRsJMf/Qq39mOR2\\nSpW+hr4jCoHeRVYLgsbggtrevGmILAlNoqCMpGZ6vDmJpq6ECV9olliDvpPgWOP+\\nm" + + "YPDreFBGxWvQrADNbRt2dmGsrsCgYEAyUHqB2wvJHFqdmeBsaacewzV8x9WgmeX\\ngUIi9REwXlGDW0Mz50dxpxcK" + + "CAYn65+7TCnY5O/jmL0VRxU1J2mSWyWTo1C+17L0\\n3fUqjxL1pkefwecxwecvC+gFFYdJ4CQ/MHHXU81Lwl1iWdF" + + "Cd2UoGddYaOF+KNeM\\nHC7cmqra+JsCgYEAlUNywzq8nUg7282E+uICfCB0LfwejuymR93CtsFgb7cRd6ak\\nECR" + + "8FGfCpH8ruWJINllbQfcHVCX47ndLZwqv3oVFKh6pAS/vVI4dpOepP8++7y1u\\ncoOvtreXCX6XqfrWDtKIvv0vjl" + + "HBhhhp6mCcRpdQjV38H7JsyJ7lih/oNjECgYAt\\nkndj5uNl5SiuVxHFhcZJO+XWf6ofLUregtevZakGMn8EE1uVa" + + "2AY7eafmoU/nZPT\\n00YB0TBATdCbn/nBSuKDESkhSg9s2GEKQZG5hBmL5uCMfo09z3SfxZIhJdlerreP\\nJ7gSi" + + "dI12N+EZxYd4xIJh/HFDgp7RRO87f+WJkofMQKBgGTnClK1VMaCRbJZPriw\\nEfeFCoOX75MxKwXs6xgrw4W//AYG" + + "GUjDt83lD6AZP6tws7gJ2IwY/qP7+lyhjEqN\\nHtfPZRGFkGZsdaksdlaksd323423d+15/UvrlRSFPNj1tWQmNKk" + + "XyRDW4IG1Oa2p\\nrALStNBx5Y9t0/LQnFI4w3aG\\n-----END PRIVATE KEY-----\\n\",\n" + + " \"project_id\": \"someprojectid\",\n" + + " \"client_email\": \"someclientid@developer.gserviceaccount.com\",\n" + + " \"client_id\": \"someclientid.apps.googleusercontent.com\",\n" + + " \"type\": \"service_account\",\n" + + " \"universe_domain\": \"fake.domain\"\n" + + "}"; + @BeforeClass public static void beforeClass() throws IOException { client = BigQueryReadClient.create(); @@ -806,6 +874,147 @@ public void testStructAndArraySqlTypes() throws InterruptedException, IOExceptio assertEquals(rowAssertMessage, new Utf8("abc"), structRecord.get("str_field")); } + @Test + public void testUniverseDomainWithInvalidUniverseDomain() throws IOException { + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create(loadCredentials(FAKE_JSON_CRED_WITH_GOOGLE_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + try { + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testInvalidUniverseDomainWithMismatchCredentials() throws IOException { + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder() + .setCredentialsProvider( + FixedCredentialsProvider.create( + loadCredentials(FAKE_JSON_CRED_WITH_INVALID_DOMAIN))) + .setUniverseDomain("invalid.domain") + .build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + try { + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + fail("RPCs to invalid universe domain should fail"); + } catch (UnauthenticatedException e) { + assertThat( + (e.getMessage() + .contains("does not match the universe domain found in the credentials"))) + .isTrue(); + } + localClient.close(); + } + + @Test + public void testUniverseDomainWithMatchingDomain() throws IOException { + // Test a valid domain using the default credentials and Google default universe domain. + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder().setUniverseDomain("googleapis.com").build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "bigquery-public-data", + /* datasetId = */ "samples", + /* tableId = */ "shakespeare"); + + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build(); + + long rowCount = 0; + ServerStream stream = client.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(164_656, rowCount); + localClient.close(); + } + + public void testUniverseDomain() throws IOException { + // This test is not yet part presubmit integration test as it requires the apis-tpclp.goog + // universe domain credentials. + // Test a valid read session in the universe domain gdutst. + BigQueryReadSettings bigQueryReadSettings = + BigQueryReadSettings.newBuilder().setUniverseDomain("apis-tpclp.goog").build(); + BigQueryReadClient localClient = BigQueryReadClient.create(bigQueryReadSettings); + + String table = + BigQueryResource.FormatTableResource( + /* projectId = */ "google-tpc-testing-environment:cloudsdk-test-project", + /* datasetId = */ "tpc_demo_dataset", + /* tableId = */ "new_table"); + + ReadSession session = + localClient.createReadSession( + /* parent = */ parentProjectId, + /* readSession = */ ReadSession.newBuilder() + .setTable(table) + .setDataFormat(DataFormat.AVRO) + .build(), + /* maxStreamCount = */ 1); + + ReadRowsRequest readRowsRequest = + ReadRowsRequest.newBuilder().setReadStream(session.getStreams(0).getName()).build(); + + long rowCount = 0; + ServerStream stream = localClient.readRowsCallable().call(readRowsRequest); + for (ReadRowsResponse response : stream) { + rowCount += response.getRowCount(); + } + + assertEquals(1, rowCount); + localClient.close(); + } + /** * Reads to the specified row offset within the stream. If the stream does not have the desired * rows to read, it will read all of them. @@ -962,4 +1171,14 @@ private Job RunQueryJobAndExpectSuccess(QueryJobConfiguration configuration) return completedJob; } + + static GoogleCredentials loadCredentials(String credentialFile) { + try { + InputStream keyStream = new ByteArrayInputStream(credentialFile.getBytes()); + return GoogleCredentials.fromStream(keyStream); + } catch (IOException e) { + fail("Couldn't create fake JSON credentials."); + } + return null; + } }