feat(csharp/src/Drivers/BigQuery): Enhanced tracing and large resultset improvements (#3022)

- Adds tracing for the BigQuery driver
- Modifies the behavior for large result sets:
- Uses a default dataset ID for large result sets if one is not
specified.
- The caller can specify their own dataset. If it does not exist, the
driver will attempt to create it.

---------

Co-authored-by: David Coe <>

Author: davidhcoe

csharp/src/Apache.Arrow.Adbc/Tracing/ActivityExtensions.cs

@@ -63,6 +63,25 @@ public static class ActivityExtensions
             return activity?.AddTag(key, value());
         }
 
+        /// <summary>
+        /// Update the Activity to have a tag with an additional 'key' and value 'value'.
+        /// This shows up in the <see cref="TagObjects"/> enumeration. It is meant for information that
+        /// is useful to log but not needed for runtime control (for the latter, <see cref="Baggage"/>)
+        /// </summary>
+        /// <returns><see langword="this" /> for convenient chaining.</returns>
+        /// <param name="key">The tag key name as a function</param>
+        /// <param name="value">The tag value mapped to the input key</param>
+        /// /// <param name="condition">The condition to check before adding the tag</param>
+        public static Activity? AddConditionalTag(this Activity? activity, string key, string? value, bool condition)
+        {
+            if (condition)
+            {
+                return activity?.AddTag(key, value);
+            }
+
+            return activity;
+        }
+
         /// <summary>
         /// Update the Activity to have a tag with an additional 'key' and value 'value'.
         /// This shows up in the <see cref="TagObjects"/> enumeration. It is meant for information that

csharp/src/Drivers/BigQuery/ActivityExtensions.cs

@@ -0,0 +1,51 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements.  See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+using System.Diagnostics;
+using Apache.Arrow.Adbc.Tracing;
+
+namespace Apache.Arrow.Adbc.Drivers.BigQuery
+{
+    internal static class ActivityExtensions
+    {
+        private const string bigQueryKeyPrefix = "adbc.bigquery.tracing.";
+        private const string bigQueryParameterKeyValueSuffix = ".value";
+
+        public static Activity AddBigQueryTag(this Activity activity, string key, object? value)
+        {
+            string bigQueryKey = bigQueryKeyPrefix + key;
+            return activity.AddTag(bigQueryKey, value);
+        }
+
+        public static Activity AddConditionalBigQueryTag(this Activity activity, string key, string? value, bool condition)
+        {
+            string bigQueryKey = bigQueryKeyPrefix + key;
+            return activity.AddConditionalTag(key, value, condition)!;
+        }
+
+        public static Activity AddBigQueryParameterTag(this Activity activity, string parameterName, object? value)
+        {
+            if (BigQueryParameters.IsSafeToLog(parameterName))
+            {
+                string bigQueryParameterValueKey = parameterName + bigQueryParameterKeyValueSuffix;
+                return activity.AddTag(bigQueryParameterValueKey, value);
+            }
+
+            return activity;
+        }
+    }
+}

csharp/src/Drivers/BigQuery/BigQueryConnection.cs

@@ -15,6 +15,9 @@
  * limitations under the License.
  */
 
+using System;
+using System.Collections.Generic;
+
 namespace Apache.Arrow.Adbc.Drivers.BigQuery
 {
     /// <summary>
@@ -23,29 +26,48 @@ namespace Apache.Arrow.Adbc.Drivers.BigQuery
     internal class BigQueryParameters
     {
         public const string AccessToken = "adbc.bigquery.access_token";
+        public const string AllowLargeResults = "adbc.bigquery.allow_large_results";
         public const string AudienceUri = "adbc.bigquery.audience_uri";
-        public const string ProjectId = "adbc.bigquery.project_id";
+        public const string AuthenticationType = "adbc.bigquery.auth_type";
         public const string BillingProjectId = "adbc.bigquery.billing_project_id";
         public const string ClientId = "adbc.bigquery.client_id";
         public const string ClientSecret = "adbc.bigquery.client_secret";
-        public const string RefreshToken = "adbc.bigquery.refresh_token";
-        public const string AuthenticationType = "adbc.bigquery.auth_type";
+        public const string ClientTimeout = "adbc.bigquery.client.timeout";
+        public const string EvaluationKind = "adbc.bigquery.multiple_statement.evaluation_kind";
+        public const string GetQueryResultsOptionsTimeout = "adbc.bigquery.get_query_results_options.timeout";
+        public const string IncludeConstraintsWithGetObjects = "adbc.bigquery.include_constraints_getobjects";
+        public const string IncludePublicProjectId = "adbc.bigquery.include_public_project_id";
         public const string JsonCredential = "adbc.bigquery.auth_json_credential";
-        public const string AllowLargeResults = "adbc.bigquery.allow_large_results";
-        public const string LargeResultsDestinationTable = "adbc.bigquery.large_results_destination_table";
-        public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql";
         public const string LargeDecimalsAsString = "adbc.bigquery.large_decimals_as_string";
-        public const string Scopes = "adbc.bigquery.scopes";
-        public const string IncludeConstraintsWithGetObjects = "adbc.bigquery.include_constraints_getobjects";
-        public const string ClientTimeout = "adbc.bigquery.client.timeout";
+        public const string LargeResultsDataset = "adbc.bigquery.large_results_dataset";
+        public const string LargeResultsDestinationTable = "adbc.bigquery.large_results_destination_table";
+        public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency";
         public const string MaximumRetryAttempts = "adbc.bigquery.maximum_retries";
+        public const string ProjectId = "adbc.bigquery.project_id";
+        public const string RefreshToken = "adbc.bigquery.refresh_token";
         public const string RetryDelayMs = "adbc.bigquery.retry_delay_ms";
-        public const string GetQueryResultsOptionsTimeout = "adbc.bigquery.get_query_results_options.timeout";
-        public const string MaxFetchConcurrency = "adbc.bigquery.max_fetch_concurrency";
-        public const string IncludePublicProjectId = "adbc.bigquery.include_public_project_id";
-        public const string StatementType = "adbc.bigquery.multiple_statement.statement_type";
+        public const string Scopes = "adbc.bigquery.scopes";
         public const string StatementIndex = "adbc.bigquery.multiple_statement.statement_index";
-        public const string EvaluationKind = "adbc.bigquery.multiple_statement.evaluation_kind";
+        public const string StatementType = "adbc.bigquery.multiple_statement.statement_type";
+        public const string UseLegacySQL = "adbc.bigquery.use_legacy_sql";
+
+        // these values are safe to log any time
+        private static HashSet<string> safeToLog = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+        {
+            AllowLargeResults, AuthenticationType, BillingProjectId, ClientId, ClientTimeout,
+            EvaluationKind, GetQueryResultsOptionsTimeout, IncludeConstraintsWithGetObjects,
+            IncludePublicProjectId, LargeDecimalsAsString, LargeResultsDataset, LargeResultsDestinationTable,
+            MaxFetchConcurrency, MaximumRetryAttempts, ProjectId, RetryDelayMs, StatementIndex,
+            StatementType, UseLegacySQL
+        };
+
+        public static bool IsSafeToLog(string name)
+        {
+            if (safeToLog.Contains(name))
+                return true;
+
+            return false;
+        }
     }
 
     /// <summary>
@@ -68,5 +90,11 @@ internal class BigQueryConstants
 
         // default value per https://pkg.go.dev/cloud.google.com/go/bigquery#section-readme
         public const string DetectProjectId = "*detect-project-id*";
+
+        // Reuse what the ODBC driver already has in place, in case a caller
+        // has permission issues trying to create a new dataset
+        public const string DefaultLargeDatasetId = "_bqodbc_temp_tables";
+
+        public const string PublicProjectId = "bigquery-public-data";
     }
 }