From 060cf09fc1e5ce8ac340e1397e95865e3794b6e3 Mon Sep 17 00:00:00 2001 From: David Coe Date: Thu, 1 Feb 2024 12:48:08 -0500 Subject: [PATCH 1/7] fix large results --- .../Drivers/BigQuery/BigQueryConnection.cs | 34 +++++++++++++++---- .../Drivers/BigQuery/BigQueryParameters.cs | 1 + .../src/Drivers/BigQuery/BigQueryStatement.cs | 29 ++++++++++++++++ ...e.Arrow.Adbc.Tests.Drivers.BigQuery.csproj | 1 - .../BigQuery/BigQueryTestConfiguration.cs | 10 ++++++ .../Drivers/BigQuery/BigQueryTestingUtils.cs | 11 ++++++ csharp/test/Drivers/BigQuery/ClientTests.cs | 1 + ....Arrow.Adbc.Tests.Drivers.FlightSql.csproj | 1 - 8 files changed, 80 insertions(+), 8 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index ef68ef4e34..c6a216e486 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -16,6 +16,7 @@ */ using System; +using System.Collections; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics; @@ -270,6 +271,23 @@ public override IArrowArrayStream GetObjects( return new BigQueryInfoArrowStream(StandardSchemas.GetObjectsSchema, dataArrays); } + /// + /// Executes the query using the BigQueryClient. + /// + /// The query to execute. + /// Parameters to include. + /// Additional query options. + /// Additional result options. + /// + /// + /// Can later add logging or metrics around query calls. + /// + private BigQueryResults? ExecuteQuery(string sql, IEnumerable? parameters, QueryOptions? queryOptions = null, GetQueryResultsOptions? resultsOptions = null) + { + BigQueryResults? result = this.client?.ExecuteQuery(sql, parameters, queryOptions, resultsOptions); + return result; + } + private List GetCatalogs( GetObjectsDepth depth, string catalogPattern, @@ -405,7 +423,7 @@ private StructArray GetTableSchemas( } } - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); if (result != null) { @@ -481,7 +499,7 @@ private StructArray GetColumnSchema( query = string.Concat(query, string.Format("AND column_name LIKE '{0}'", Sanitize(columnNamePattern))); } - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); if (result != null) { @@ -570,7 +588,7 @@ private StructArray GetConstraintSchema( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(table)); - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); if (result != null) { @@ -631,7 +649,7 @@ private StringArray GetConstraintColumnNames( StringArray.Builder constraintColumnNamesBuilder = new StringArray.Builder(); - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); if (result != null) { @@ -661,7 +679,7 @@ private StructArray GetConstraintsUsage( string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE WHERE constraint_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(constraintName)); - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); if (result != null) { @@ -788,7 +806,7 @@ public override Schema GetTableSchema(string catalog, string dbSchema, string ta string query = string.Format("SELECT * FROM `{0}`.`{1}`.INFORMATION_SCHEMA.COLUMNS WHERE table_name = '{2}'", Sanitize(catalog), Sanitize(dbSchema), Sanitize(tableName)); - BigQueryResults? result = this.client?.ExecuteQuery(query, parameters: null); + BigQueryResults? result = ExecuteQuery(query, parameters: null); List fields = new List(); @@ -1014,6 +1032,10 @@ private IReadOnlyDictionary ParseOptions() { options[keyValuePair.Key] = keyValuePair.Value; } + if(keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) + { + options[keyValuePair.Key] = keyValuePair.Value; + } } return new ReadOnlyDictionary(options); diff --git a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs index ff05036dc4..ff57ea8faa 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryParameters.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryParameters.cs @@ -29,6 +29,7 @@ public class BigQueryParameters public const string AuthenticationType = "adbc.bigquery.auth_type"; public const string JsonCredential = "adbc.bigquery.auth_json_credential"; public const string AllowLargeResults = "adbc.bigquery.allow_large_results"; + public const string LargeResultsDestinationTable = "adbc.bigquery.large_results_destination_table"; public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql"; public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string"; public const string Scopes = "adbc.bigquery.scopes"; diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 389e462e55..897fb2d5ee 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -18,6 +18,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.ComponentModel; using System.Data.SqlTypes; using System.IO; using System.Linq; @@ -27,6 +28,7 @@ using Apache.Arrow.Ipc; using Apache.Arrow.Types; using Google.Apis.Auth.OAuth2; +using Google.Apis.Bigquery.v2.Data; using Google.Cloud.BigQuery.Storage.V1; using Google.Cloud.BigQuery.V2; using TableFieldSchema = Google.Apis.Bigquery.v2.Data.TableFieldSchema; @@ -185,6 +187,33 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName) { options.AllowLargeResults = true ? keyValuePair.Value.ToLower().Equals("true") : false; } + if(keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) + { + string destinationTable = keyValuePair.Value; + + if (!destinationTable.Contains(".")) + throw new InvalidOperationException($"{BigQueryParameters.LargeResultsDestinationTable} is invalid"); + + string projectId = string.Empty; + string datasetId = string.Empty; + string tableId = string.Empty; + + List segments = destinationTable.Split('.').Where(x => x.Length > 0).ToList(); + + if(segments.Count != 3) + throw new InvalidOperationException($"{BigQueryParameters.LargeResultsDestinationTable} cannot be parsed"); + + projectId = segments[0]; + datasetId = segments[1]; + tableId = segments[2]; + + options.DestinationTable = new TableReference() + { + ProjectId = projectId, + DatasetId = datasetId, + TableId = tableId + }; + } if (keyValuePair.Key == BigQueryParameters.UseLegacySQL) { options.UseLegacySql = true ? keyValuePair.Value.ToLower().Equals("true") : false; diff --git a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj index 74712cdc2f..e2d7921490 100644 --- a/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj +++ b/csharp/test/Drivers/BigQuery/Apache.Arrow.Adbc.Tests.Drivers.BigQuery.csproj @@ -13,7 +13,6 @@ - diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index a6b0ab6b07..b81a60d186 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -24,6 +24,11 @@ namespace Apache.Arrow.Adbc.Tests.Drivers.BigQuery /// internal class BigQueryTestConfiguration : TestConfiguration { + public BigQueryTestConfiguration() + { + AllowLargeResults = true; + } + [JsonPropertyName("projectId")] public string ProjectId { get; set; } @@ -42,5 +47,10 @@ internal class BigQueryTestConfiguration : TestConfiguration [JsonPropertyName("jsonCredential")] public string JsonCredential { get; set; } + [JsonPropertyName("allowLargeResults")] + public bool AllowLargeResults { get; set; } + + [JsonPropertyName("largeResultsDestinationTable")] + public string LargeResultsDestinationTable { get; set; } } } diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 874ccac791..8459dd23fd 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -15,6 +15,7 @@ * limitations under the License. */ +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -74,6 +75,16 @@ internal static Dictionary GetBigQueryParameters(BigQueryTestConf parameters.Add(BigQueryParameters.Scopes, testConfiguration.Scopes); } + if(testConfiguration.AllowLargeResults) + { + parameters.Add(BigQueryParameters.AllowLargeResults, testConfiguration.AllowLargeResults.ToString()); + } + + if(!string.IsNullOrEmpty(testConfiguration.LargeResultsDestinationTable)) + { + parameters.Add(BigQueryParameters.LargeResultsDestinationTable, testConfiguration.LargeResultsDestinationTable); + } + return parameters; } diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index 94b28833dd..2633240824 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Xunit; using Xunit; diff --git a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj index 9036f5acfd..7a45c99b00 100644 --- a/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj +++ b/csharp/test/Drivers/FlightSql/Apache.Arrow.Adbc.Tests.Drivers.FlightSql.csproj @@ -15,7 +15,6 @@ - From 862006bbabaa758baeed8df3ceb62b3e9d69bfef Mon Sep 17 00:00:00 2001 From: David Coe Date: Thu, 1 Feb 2024 12:53:35 -0500 Subject: [PATCH 2/7] cleanup --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 1 - csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs | 2 +- csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs | 1 - csharp/test/Drivers/BigQuery/ClientTests.cs | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 897fb2d5ee..8da33469ae 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -18,7 +18,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.ComponentModel; using System.Data.SqlTypes; using System.IO; using System.Linq; diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs index b81a60d186..a7a5622929 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestConfiguration.cs @@ -26,7 +26,7 @@ internal class BigQueryTestConfiguration : TestConfiguration { public BigQueryTestConfiguration() { - AllowLargeResults = true; + AllowLargeResults = false; } [JsonPropertyName("projectId")] diff --git a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs index 8459dd23fd..ffca233cc5 100644 --- a/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs +++ b/csharp/test/Drivers/BigQuery/BigQueryTestingUtils.cs @@ -15,7 +15,6 @@ * limitations under the License. */ -using System; using System.Collections.Generic; using System.IO; using System.Linq; diff --git a/csharp/test/Drivers/BigQuery/ClientTests.cs b/csharp/test/Drivers/BigQuery/ClientTests.cs index 2633240824..94b28833dd 100644 --- a/csharp/test/Drivers/BigQuery/ClientTests.cs +++ b/csharp/test/Drivers/BigQuery/ClientTests.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using Apache.Arrow.Adbc.Drivers.BigQuery; using Apache.Arrow.Adbc.Tests.Xunit; using Xunit; From 52c4fc08d6887f91b73e483cde420505affdd8e8 Mon Sep 17 00:00:00 2001 From: David Coe Date: Thu, 1 Feb 2024 14:03:25 -0500 Subject: [PATCH 3/7] update docs --- csharp/src/Drivers/BigQuery/readme.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 611374c585..79553a41e3 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -49,6 +49,10 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G **adbc.bigquery.auth_json_credential**
    Required if using `service` authentication. This value is passed to the [GoogleCredential.FromJson](https://cloud.google.com/dotnet/docs/reference/Google.Apis/latest/Google.Apis.Auth.OAuth2.GoogleCredential#Google_Apis_Auth_OAuth2_GoogleCredential_FromJson_System_String) method. +**adbc.bigquery.large_results_destination_table**
+    Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions to if configured. Expected the format to be in `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. + + **adbc.bigquery.project_id**
    The [Project ID](https://cloud.google.com/resource-manager/docs/creating-managing-projects) used for accessing BigQuery. From 55961fe1f69243fe0ac8ff988d3a3c976e98393c Mon Sep 17 00:00:00 2001 From: David Coe Date: Thu, 1 Feb 2024 14:08:03 -0500 Subject: [PATCH 4/7] update docs --- csharp/src/Drivers/BigQuery/readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/readme.md b/csharp/src/Drivers/BigQuery/readme.md index 79553a41e3..847513c485 100644 --- a/csharp/src/Drivers/BigQuery/readme.md +++ b/csharp/src/Drivers/BigQuery/readme.md @@ -50,7 +50,7 @@ https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/G     Required if using `service` authentication. This value is passed to the [GoogleCredential.FromJson](https://cloud.google.com/dotnet/docs/reference/Google.Apis/latest/Google.Apis.Auth.OAuth2.GoogleCredential#Google_Apis_Auth_OAuth2_GoogleCredential_FromJson_System_String) method. **adbc.bigquery.large_results_destination_table**
-    Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions to if configured. Expected the format to be in `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. +    Sets the [DestinationTable](https://cloud.google.com/dotnet/docs/reference/Google.Cloud.BigQuery.V2/latest/Google.Cloud.BigQuery.V2.QueryOptions#Google_Cloud_BigQuery_V2_QueryOptions_DestinationTable) value of the QueryOptions if configured. Expects the format to be `{projectId}.{datasetId}.{tableId}` to set the corresponding values in the [TableReference](https://github.com/googleapis/google-api-dotnet-client/blob/6c415c73788b848711e47c6dd33c2f93c76faf97/Src/Generated/Google.Apis.Bigquery.v2/Google.Apis.Bigquery.v2.cs#L9348) class. **adbc.bigquery.project_id**
From 159396f92a3cbb266c80a9263eb2904a716aced7 Mon Sep 17 00:00:00 2001 From: David Coe Date: Fri, 2 Feb 2024 14:32:08 -0500 Subject: [PATCH 5/7] PR feedback --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 8da33469ae..3bbd04a616 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -186,7 +186,7 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName) { options.AllowLargeResults = true ? keyValuePair.Value.ToLower().Equals("true") : false; } - if(keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) + if (keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) { string destinationTable = keyValuePair.Value; From 9fc9561025fbbe5c074ee574fb348d791ed5c7b6 Mon Sep 17 00:00:00 2001 From: David Coe Date: Fri, 2 Feb 2024 14:34:38 -0500 Subject: [PATCH 6/7] PR feedback --- csharp/src/Drivers/BigQuery/BigQueryConnection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs index c6a216e486..a06b72eb0e 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryConnection.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryConnection.cs @@ -1032,7 +1032,7 @@ private IReadOnlyDictionary ParseOptions() { options[keyValuePair.Key] = keyValuePair.Value; } - if(keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) + if (keyValuePair.Key == BigQueryParameters.LargeResultsDestinationTable) { options[keyValuePair.Key] = keyValuePair.Value; } From ae6d1aad812fda7632fa54ab8047f1acf67c86fd Mon Sep 17 00:00:00 2001 From: David Coe Date: Sat, 3 Feb 2024 11:32:29 -0500 Subject: [PATCH 7/7] PR feedback --- csharp/src/Drivers/BigQuery/BigQueryStatement.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs index 3bbd04a616..f4f21740a6 100644 --- a/csharp/src/Drivers/BigQuery/BigQueryStatement.cs +++ b/csharp/src/Drivers/BigQuery/BigQueryStatement.cs @@ -197,15 +197,18 @@ static IArrowReader ReadChunk(BigQueryReadClient readClient, string streamName) string datasetId = string.Empty; string tableId = string.Empty; - List segments = destinationTable.Split('.').Where(x => x.Length > 0).ToList(); + string[] segments = destinationTable.Split('.'); - if(segments.Count != 3) + if(segments.Length != 3) throw new InvalidOperationException($"{BigQueryParameters.LargeResultsDestinationTable} cannot be parsed"); projectId = segments[0]; datasetId = segments[1]; tableId = segments[2]; + if(string.IsNullOrEmpty(projectId.Trim()) || string.IsNullOrEmpty(datasetId.Trim()) || string.IsNullOrEmpty(tableId.Trim())) + throw new InvalidOperationException($"{BigQueryParameters.LargeResultsDestinationTable} contains invalid values"); + options.DestinationTable = new TableReference() { ProjectId = projectId,