Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bigquery): add range partitioning to tables, load jobs, and query jobs #9477

Merged
merged 7 commits into from
Oct 18, 2019
+487 −11
Merged

feat(bigquery): add range partitioning to tables, load jobs, and query jobs #9477

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
2fed3bb
feat(bigquery): add range partitioning to tables, load jobs, and quer…
tswast Oct 14, 2019
5b27f80
Merge remote-tracking branch 'upstream/master' into bq-range-partitio…
tswast Oct 15, 2019
eacb9d8
Add integer range partitioning classes to bigquery module
tswast Oct 15, 2019
f04153f
Merge remote-tracking branch 'upstream/master' into bq-range-partitio…
tswast Oct 17, 2019
9bb5d8b
Rename PartitionRange to RangeDefinition
tswast Oct 17, 2019
2cc6443
Revert "Rename PartitionRange to RangeDefinition"
tswast Oct 17, 2019
e1225ee
Add Beta disclaimer to range partitioning features.
tswast Oct 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions bigquery/docs/reference.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,13 @@ Table
.. autosummary::
:toctree: generated

table.PartitionRange
table.RangePartitioning
table.Row
table.RowIterator
table.Table
table.TableListItem
table.TableReference
table.Row
table.RowIterator
table.TimePartitioning
table.TimePartitioningType

Expand Down
11 changes: 8 additions & 3 deletions bigquery/google/cloud/bigquery/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,9 +73,11 @@
from google.cloud.bigquery.routine import RoutineArgument
from google.cloud.bigquery.routine import RoutineReference
from google.cloud.bigquery.schema import SchemaField
from google.cloud.bigquery.table import PartitionRange
from google.cloud.bigquery.table import RangePartitioning
from google.cloud.bigquery.table import Row
from google.cloud.bigquery.table import Table
from google.cloud.bigquery.table import TableReference
from google.cloud.bigquery.table import Row
from google.cloud.bigquery.table import TimePartitioningType
from google.cloud.bigquery.table import TimePartitioning
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
Expand All @@ -96,16 +98,19 @@
# Tables
"Table",
"TableReference",
"PartitionRange",
"RangePartitioning",
"Row",
"TimePartitioning",
"TimePartitioningType",
# Jobs
"CopyJob",
"CopyJobConfig",
"ExtractJob",
"ExtractJobConfig",
"LoadJob",
"LoadJobConfig",
"UnknownJob",
"TimePartitioningType",
"TimePartitioning",
# Models
"Model",
"ModelReference",
Expand Down
105 changes: 101 additions & 4 deletions bigquery/google/cloud/bigquery/job.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@
from google.cloud.bigquery.dataset import Dataset
from google.cloud.bigquery.dataset import DatasetListItem
from google.cloud.bigquery.dataset import DatasetReference
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

cleanup from another PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, recently added for models encryption but I forgot to ask for the imports to be alphabetized.

from google.cloud.bigquery.external_config import ExternalConfig
from google.cloud.bigquery import _helpers
from google.cloud.bigquery.query import _query_param_from_api_repr
from google.cloud.bigquery.query import ArrayQueryParameter
from google.cloud.bigquery.query import ScalarQueryParameter
Expand All @@ -37,12 +39,11 @@
from google.cloud.bigquery.routine import RoutineReference
from google.cloud.bigquery.schema import SchemaField
from google.cloud.bigquery.table import _EmptyRowIterator
from google.cloud.bigquery.table import RangePartitioning
from google.cloud.bigquery.table import _table_arg_to_table_ref
from google.cloud.bigquery.table import TableReference
from google.cloud.bigquery.table import Table
from google.cloud.bigquery.table import TimePartitioning
from google.cloud.bigquery import _helpers
from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration

_DONE_STATE = "DONE"
_STOPPED_REASON = "stopped"
Expand Down Expand Up @@ -1180,6 +1181,40 @@ def quote_character(self):
def quote_character(self, value):
self._set_sub_prop("quote", value)

@property
def range_partitioning(self):
"""Optional[google.cloud.bigquery.table.RangePartitioning]:
Configures range-based partitioning for destination table.

.. note::
**Beta**. The integer range partitioning feature is in a
pre-release state and might change or have limited support.

Only specify at most one of
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

Raises:
ValueError:
If the value is not
:class:`~google.cloud.bigquery.table.RangePartitioning` or
:data:`None`.
"""
resource = self._get_sub_prop("rangePartitioning")
if resource is not None:
return RangePartitioning(_properties=resource)

@range_partitioning.setter
def range_partitioning(self, value):
resource = value
if isinstance(value, RangePartitioning):
resource = value._properties
elif value is not None:
raise ValueError(
"Expected value to be RangePartitioning or None, got {}.".format(value)
)
self._set_sub_prop("rangePartitioning", resource)

@property
def schema(self):
"""List[google.cloud.bigquery.schema.SchemaField]: Schema of the
Expand Down Expand Up @@ -1249,6 +1284,10 @@ def source_format(self, value):
def time_partitioning(self):
"""google.cloud.bigquery.table.TimePartitioning: Specifies time-based
partitioning for the destination table.

Only specify at most one of
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
"""
prop = self._get_sub_prop("timePartitioning")
if prop is not None:
Expand Down Expand Up @@ -1463,6 +1502,13 @@ def destination_table_friendly_name(self):
"""
return self._configuration.destination_table_friendly_name

@property
def range_partitioning(self):
"""See
:attr:`google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.
"""
return self._configuration.range_partitioning

@property
def time_partitioning(self):
"""See
Expand Down Expand Up @@ -2242,6 +2288,40 @@ def query_parameters(self):
def query_parameters(self, values):
self._set_sub_prop("queryParameters", _to_api_repr_query_parameters(values))

@property
def range_partitioning(self):
"""Optional[google.cloud.bigquery.table.RangePartitioning]:
Configures range-based partitioning for destination table.

.. note::
**Beta**. The integer range partitioning feature is in a
pre-release state and might change or have limited support.

Only specify at most one of
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

Raises:
ValueError:
If the value is not
:class:`~google.cloud.bigquery.table.RangePartitioning` or
:data:`None`.
"""
resource = self._get_sub_prop("rangePartitioning")
if resource is not None:
return RangePartitioning(_properties=resource)

@range_partitioning.setter
def range_partitioning(self, value):
resource = value
if isinstance(value, RangePartitioning):
resource = value._properties
elif value is not None:
raise ValueError(
"Expected value to be RangePartitioning or None, got {}.".format(value)
)
self._set_sub_prop("rangePartitioning", resource)

@property
def udf_resources(self):
"""List[google.cloud.bigquery.query.UDFResource]: user
Expand Down Expand Up @@ -2318,8 +2398,18 @@ def table_definitions(self, values):

@property
def time_partitioning(self):
"""google.cloud.bigquery.table.TimePartitioning: Specifies time-based
partitioning for the destination table.
"""Optional[google.cloud.bigquery.table.TimePartitioning]: Specifies
time-based partitioning for the destination table.

Only specify at most one of
:attr:`~google.cloud.bigquery.job.LoadJobConfig.time_partitioning` or
:attr:`~google.cloud.bigquery.job.LoadJobConfig.range_partitioning`.

Raises:
ValueError:
If the value is not
:class:`~google.cloud.bigquery.table.TimePartitioning` or
:data:`None`.
"""
prop = self._get_sub_prop("timePartitioning")
if prop is not None:
Expand Down Expand Up @@ -2552,6 +2642,13 @@ def maximum_bytes_billed(self):
"""
return self._configuration.maximum_bytes_billed

@property
def range_partitioning(self):
"""See
:attr:`google.cloud.bigquery.job.QueryJobConfig.range_partitioning`.
"""
return self._configuration.range_partitioning

@property
def table_definitions(self):
"""See
Expand Down
Loading