From 16cf7b65d5b73a2f7fea2894768cb4456a86929d Mon Sep 17 00:00:00 2001 From: Vishal Karve Date: Mon, 7 Aug 2023 11:51:04 +0530 Subject: [PATCH] address comments --- README-template.md | 27 ++++++++++++++----- .../connector/common/BigQueryClient.java | 4 +-- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/README-template.md b/README-template.md index 51b3bdee0..70689ebcd 100644 --- a/README-template.md +++ b/README-template.md @@ -274,7 +274,7 @@ df.write \ Writing to existing partitioned tables (date partitioned, ingestion time partitioned and range partitioned) in APPEND save mode is fully supported by the connector and the BigQuery Storage Write -API. Partition overwrite and the use of `datePartition`, `partitionField` and `partitionType` as +API. Partition overwrite and the use of `datePartition`, `partitionField`, `partitionType`, `partitionRangeStart`, `partitionRangeEnd`, `partitionRangeInterval` as described below is not supported at this moment by the direct write method. **Important:** Please refer to the [data ingestion pricing](https://cloud.google.com/bigquery/pricing#data_ingestion_pricing) @@ -618,10 +618,12 @@ word-break:break-word partitionField - If field is specified together with `partitionType`, the table is partitioned by this field. - The field must be a top-level TIMESTAMP or DATE field. Its mode must be NULLABLE + If this field is specified, the table is partitioned by this field. +
For Time partitioning, specify together with the option `partitionType`. +
For Integer-range partitioning, specify together with the 3 options: `partitionRangeStart`, `partitionRangeEnd, `partitionRangeInterval`. +
The field must be a top-level TIMESTAMP or DATE field for Time partitioning, or INT64 for Integer-range partitioning. Its mode must be NULLABLE or REQUIRED. - If the option is not set for a partitioned table, then the table will be partitioned by pseudo + If the option is not set for a Time partitioned table, then the table will be partitioned by pseudo column, referenced via either'_PARTITIONTIME' as TIMESTAMP type, or '_PARTITIONDATE' as DATE type.
(Optional). @@ -642,13 +644,26 @@ word-break:break-word partitionType - Supported types are: HOUR, DAY, MONTH, YEAR -
This option is mandatory for a target table to be partitioned. + Used to specify Time partitioning. +
Supported types are: HOUR, DAY, MONTH, YEAR +
This option is mandatory for a target table to be Time partitioned.
(Optional. Defaults to DAY if PartitionField is specified).
Not supported by the `DIRECT` write method. Write + + partitionRangeStart, + partitionRangeEnd, + partitionRangeInterval + + Used to specify Integer-range partitioning. +
These options are mandatory for a target table to be Integer-range partitioned. +
All 3 options must be specified. +
Not supported by the `DIRECT` write method. + + Write + clusteredFields diff --git a/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryClient.java b/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryClient.java index 3e9599106..80528fa5e 100644 --- a/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryClient.java +++ b/bigquery-connector-common/src/main/java/com/google/cloud/bigquery/connector/common/BigQueryClient.java @@ -607,8 +607,8 @@ public void loadDataIntoTable( } if (options.getPartitionField().isPresent() && options.getPartitionRange().isPresent()) { RangePartitioning.Builder rangePartitionBuilder = RangePartitioning.newBuilder(); - rangePartitionBuilder.setField(options.getPartitionField().get()); - rangePartitionBuilder.setRange(options.getPartitionRange().get()); + options.getPartitionField().ifPresent(rangePartitionBuilder::setField); + options.getPartitionRange().ifPresent(rangePartitionBuilder::setRange); jobConfiguration.setRangePartitioning(rangePartitionBuilder.build()); }