chore: sync v3 branch with main

.github/.OwlBot.lock.yaml

@@ -1,3 +1,16 @@
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 docker:
   image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest
-  digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828
+  digest: sha256:7cffbc10910c3ab1b852c05114a08d374c195a81cdec1d4a67a1d129331d0bfe

.github/sync-repo-settings.yaml

@@ -14,7 +14,6 @@ branchProtectionRules:
     - 'Kokoro snippets-3.8'
     - 'cla/google'
     - 'Samples - Lint'
-    - 'Samples - Python 3.6'
     - 'Samples - Python 3.7'
     - 'Samples - Python 3.8'
 - pattern: v3
@@ -26,6 +25,5 @@ branchProtectionRules:
     - 'Kokoro snippets-3.8'
     - 'cla/google'
     - 'Samples - Lint'
-    - 'Samples - Python 3.6'
     - 'Samples - Python 3.7'
     - 'Samples - Python 3.8'

.kokoro/continuous/prerelease-deps-3.10.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "prerelease_deps-3.10"
+}

.kokoro/continuous/prerelease-deps-3.8.cfg

@@ -3,5 +3,5 @@
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
-    value: "prerelease_deps"
+    value: "prerelease_deps-3.8"
 }

.kokoro/presubmit/prerelease-deps-3.10.cfg

@@ -0,0 +1,7 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+
+# Only run this nox session.
+env_vars: {
+    key: "NOX_SESSION"
+    value: "prerelease_deps-3.10"
+}

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
     -   id: end-of-file-fixer
     -   id: check-yaml
 -   repo: https://github.com/psf/black
-    rev: 19.10b0
+    rev: 22.3.0
     hooks:
     - id: black
 -   repo: https://gitlab.com/pycqa/flake8

CHANGELOG.md

@@ -5,6 +5,54 @@
 [1]: https://pypi.org/project/google-cloud-bigquery/#history
 
 
+### [2.34.3](https://github.com/googleapis/python-bigquery/compare/v2.34.2...v2.34.3) (2022-03-29)
+
+
+### Bug Fixes
+
+* update content-type header ([#1171](https://github.com/googleapis/python-bigquery/issues/1171)) ([921b440](https://github.com/googleapis/python-bigquery/commit/921b440fdd151e88ee5b3e0d9fb90177877dc11a))
+
+### [2.34.2](https://github.com/googleapis/python-bigquery/compare/v2.34.1...v2.34.2) (2022-03-05)
+
+
+### Bug Fixes
+
+* **deps:** require google-api-core>=1.31.5, >=2.3.2 ([#1157](https://github.com/googleapis/python-bigquery/issues/1157)) ([0c15790](https://github.com/googleapis/python-bigquery/commit/0c15790720ff573a501cfe760dd74ee166e1a353))
+* **deps:** require proto-plus>=1.15.0 ([0c15790](https://github.com/googleapis/python-bigquery/commit/0c15790720ff573a501cfe760dd74ee166e1a353))
+
+### [2.34.1](https://github.com/googleapis/python-bigquery/compare/v2.34.0...v2.34.1) (2022-03-02)
+
+
+### Dependencies
+
+* add "extra" for IPython, exclude bad IPython release ([#1151](https://github.com/googleapis/python-bigquery/issues/1151)) ([0fbe12d](https://github.com/googleapis/python-bigquery/commit/0fbe12d1ababa45aa774b026a93c0af9e8f343d9))
+* allow pyarrow 7.0 ([#1112](https://github.com/googleapis/python-bigquery/issues/1112)) ([57f8ea9](https://github.com/googleapis/python-bigquery/commit/57f8ea95e152dc238e7a4941f96c54be53f7eaf3))
+
+## [2.34.0](https://github.com/googleapis/python-bigquery/compare/v2.33.0...v2.34.0) (2022-02-18)
+
+
+### Features
+
+* support BI Engine statistics in query job ([#1144](https://github.com/googleapis/python-bigquery/issues/1144)) ([7482549](https://github.com/googleapis/python-bigquery/commit/7482549cb42ed5302634ab4fb7b4efcd97b35c68))
+
+## [2.33.0](https://github.com/googleapis/python-bigquery/compare/v2.32.0...v2.33.0) (2022-02-16)
+
+
+### Features
+
+* add `--no_query_cache` option to `%%bigquery` magics to disable query cache ([#1141](https://github.com/googleapis/python-bigquery/issues/1141)) ([7dd30af](https://github.com/googleapis/python-bigquery/commit/7dd30af41b8a595b96176c964ba14aa41645ef0d))
+
+
+### Bug Fixes
+
+* return 403 when VPC-SC violation happens ([#1131](https://github.com/googleapis/python-bigquery/issues/1131)) ([f5daa9b](https://github.com/googleapis/python-bigquery/commit/f5daa9b41377a58cb3220bb2ab7c72adc6462196))
+
+
+### Documentation
+
+* reference BigQuery REST API defaults in `LoadJobConfig` descrip… ([#1132](https://github.com/googleapis/python-bigquery/issues/1132)) ([18d9580](https://github.com/googleapis/python-bigquery/commit/18d958062721d6be81e7bd7a5bd66f277344a864))
+* show common job properties in `get_job` and `cancel_job` samples ([#1137](https://github.com/googleapis/python-bigquery/issues/1137)) ([8edc10d](https://github.com/googleapis/python-bigquery/commit/8edc10d019bd96defebc4f92a47774901e9b956f))
+
 ## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12)
 
 

docs/conf.py

@@ -362,7 +362,10 @@
 intersphinx_mapping = {
     "python": ("https://python.readthedocs.org/en/latest/", None),
     "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None),
-    "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,),
+    "google.api_core": (
+        "https://googleapis.dev/python/google-api-core/latest/",
+        None,
+    ),
     "grpc": ("https://grpc.github.io/grpc/python/", None),
     "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None),
     "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None),

docs/snippets.py

@@ -752,45 +752,6 @@ def test_client_query_total_rows(client, capsys):
     assert "Got 100 rows." in out
 
 
-def test_manage_job(client):
-    sql = """
-        SELECT corpus
-        FROM `bigquery-public-data.samples.shakespeare`
-        GROUP BY corpus;
-    """
-    location = "us"
-    job = client.query(sql, location=location)
-    job_id = job.job_id
-
-    # [START bigquery_cancel_job]
-    # TODO(developer): Uncomment the lines below and replace with your values.
-    # from google.cloud import bigquery
-    # client = bigquery.Client()
-    # job_id = 'bq-job-123x456-123y123z123c'  # replace with your job ID
-    # location = 'us'                         # replace with your location
-
-    job = client.cancel_job(job_id, location=location)
-    # [END bigquery_cancel_job]
-
-    # [START bigquery_get_job]
-    # TODO(developer): Uncomment the lines below and replace with your values.
-    # from google.cloud import bigquery
-    # client = bigquery.Client()
-    # job_id = 'bq-job-123x456-123y123z123c'  # replace with your job ID
-    # location = 'us'                         # replace with your location
-
-    job = client.get_job(job_id, location=location)  # API request
-
-    # Print selected job properties
-    print("Details for job {} running in {}:".format(job_id, location))
-    print(
-        "\tType: {}\n\tState: {}\n\tCreated: {}".format(
-            job.job_type, job.state, job.created
-        )
-    )
-    # [END bigquery_get_job]
-
-
 def test_query_external_gcs_permanent_table(client, to_delete):
     dataset_id = "query_external_gcs_{}".format(_millis())
     project = client.project

google/cloud/bigquery/_pandas_helpers.py

@@ -659,7 +659,10 @@ def dataframe_to_parquet(
     bq_schema = schema._to_schema_fields(bq_schema)
     arrow_table = dataframe_to_arrow(dataframe, bq_schema)
     pyarrow.parquet.write_table(
-        arrow_table, filepath, compression=parquet_compression, **kwargs,
+        arrow_table,
+        filepath,
+        compression=parquet_compression,
+        **kwargs,
     )
 
 

google/cloud/bigquery/_tqdm_helpers.py

@@ -95,7 +95,9 @@ def wait_for_query(
             progress_bar.total = len(query_job.query_plan)
             progress_bar.set_description(
                 "Query executing stage {} and status {} : {:0.2f}s".format(
-                    current_stage.name, current_stage.status, time.time() - start_time,
+                    current_stage.name,
+                    current_stage.status,
+                    time.time() - start_time,
                 ),
             )
         try:

google/cloud/bigquery/client.py

@@ -1868,9 +1868,7 @@ def _get_query_results(
 
     def job_from_resource(
         self, resource: dict
-    ) -> Union[
-        job.CopyJob, job.ExtractJob, job.LoadJob, job.QueryJob, job.UnknownJob,
-    ]:
+    ) -> Union[job.CopyJob, job.ExtractJob, job.LoadJob, job.QueryJob, job.UnknownJob]:
         """Detect correct job type from resource and instantiate.
 
         Args:
@@ -1952,8 +1950,8 @@ def create_job(
                 timeout=timeout,
             )
         elif "extract" in job_config:
-            extract_job_config = google.cloud.bigquery.job.ExtractJobConfig.from_api_repr(
-                job_config
+            extract_job_config = (
+                google.cloud.bigquery.job.ExtractJobConfig.from_api_repr(job_config)
             )
             source = _get_sub_prop(job_config, ["extract", "sourceTable"])
             if source:
@@ -2124,7 +2122,8 @@ def cancel_job(
         job_instance = self.job_from_resource(resource["job"])  # never an UnknownJob
 
         return typing.cast(
-            Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob], job_instance,
+            Union[job.LoadJob, job.CopyJob, job.ExtractJob, job.QueryJob],
+            job_instance,
         )
 
     def list_jobs(
@@ -3319,7 +3318,14 @@ def query(
         # _default_query_job_config) up to this point.
         if api_method == enums.QueryApiMethod.QUERY:
             return _job_helpers.query_jobs_query(
-                self, query, job_config, location, project, retry, timeout, job_retry,
+                self,
+                query,
+                job_config,
+                location,
+                project,
+                retry,
+                timeout,
+                job_retry,
             )
         elif api_method == enums.QueryApiMethod.INSERT:
             return _job_helpers.query_jobs_insert(
@@ -4050,7 +4056,7 @@ def _get_upload_headers(user_agent):
         "Accept": "application/json",
         "Accept-Encoding": "gzip, deflate",
         "User-Agent": user_agent,
-        "content-type": "application/json",
+        "content-type": "application/json; charset=UTF-8",
     }
 
 

google/cloud/bigquery/dataset.py

@@ -192,7 +192,7 @@ def __repr__(self):
         )
 
     def _key(self):
-        """ A tuple key that uniquely describes this field.
+        """A tuple key that uniquely describes this field.
         Used to compute this instance's hashcode and evaluate equality.
         Returns:
             Tuple: The contents of this :class:`~google.cloud.bigquery.dataset.AccessEntry`.

google/cloud/bigquery/dbapi/_helpers.py

@@ -486,8 +486,7 @@ def raise_on_closed(
     """Make public instance methods raise an error if the instance is closed."""
 
     def _raise_on_closed(method):
-        """Make a non-static method raise an error if its containing instance is closed.
-        """
+        """Make a non-static method raise an error if its containing instance is closed."""
 
         def with_closed_check(self, *args, **kwargs):
             if getattr(self, closed_attr_name):
@@ -498,8 +497,7 @@ def with_closed_check(self, *args, **kwargs):
         return with_closed_check
 
     def decorate_public_methods(klass):
-        """Apply ``_raise_on_closed()`` decorator to public instance methods.
-        """
+        """Apply ``_raise_on_closed()`` decorator to public instance methods."""
         for name in dir(klass):
             if name.startswith("_") and name != "__iter__":
                 continue

google/cloud/bigquery/external_config.py

@@ -289,8 +289,7 @@ def from_api_repr(cls, resource: dict) -> "BigtableColumnFamily":
 
 
 class BigtableOptions(object):
-    """Options that describe how to treat Bigtable tables as BigQuery tables.
-    """
+    """Options that describe how to treat Bigtable tables as BigQuery tables."""
 
     _SOURCE_FORMAT = "BIGTABLE"
     _RESOURCE_NAME = "bigtableOptions"
@@ -557,7 +556,11 @@ def from_api_repr(cls, resource: dict) -> "GoogleSheetsOptions":
 )
 
 OptionsType = Union[
-    AvroOptions, BigtableOptions, CSVOptions, GoogleSheetsOptions, ParquetOptions,
+    AvroOptions,
+    BigtableOptions,
+    CSVOptions,
+    GoogleSheetsOptions,
+    ParquetOptions,
 ]
 
 

google/cloud/bigquery/job/base.py

@@ -45,6 +45,7 @@
     "invalidQuery": http.client.BAD_REQUEST,
     "notFound": http.client.NOT_FOUND,
     "notImplemented": http.client.NOT_IMPLEMENTED,
+    "policyViolation": http.client.FORBIDDEN,
     "quotaExceeded": http.client.FORBIDDEN,
     "rateLimitExceeded": http.client.FORBIDDEN,
     "resourceInUse": http.client.BAD_REQUEST,

google/cloud/bigquery/job/load.py

@@ -33,9 +33,21 @@
 class LoadJobConfig(_JobConfig):
     """Configuration options for load jobs.
 
-    All properties in this class are optional. Values which are :data:`None` ->
-    server defaults. Set properties on the constructed configuration by using
-    the property name as the name of a keyword argument.
+    Set properties on the constructed configuration by using the property name
+    as the name of a keyword argument. Values which are unset or :data:`None`
+    use the BigQuery REST API default values. See the `BigQuery REST API
+    reference documentation
+    <https://cloud.google.com/bigquery/docs/reference/rest/v2/Job#JobConfigurationLoad>`_
+    for a list of default values.
+
+    Required options differ based on the
+    :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` value.
+    For example, the BigQuery API's default value for
+    :attr:`~google.cloud.bigquery.job.LoadJobConfig.source_format` is ``"CSV"``.
+    When loading a CSV file, either
+    :attr:`~google.cloud.bigquery.job.LoadJobConfig.schema` must be set or
+    :attr:`~google.cloud.bigquery.job.LoadJobConfig.autodetect` must be set to
+    :data:`True`.
     """
 
     def __init__(self, **kwargs) -> None: