Skip to content

Commit

Permalink
Replace format with FormatOptions in ExternalDataConfiguration
Browse files Browse the repository at this point in the history
  • Loading branch information
@mziccard
mziccard committed Dec 2, 2015
1 parent 5f6357e commit 79001cf
Show file tree
Hide file tree
Showing 8 changed files with 200 additions and 88 deletions.
32 changes: 10 additions & 22 deletions gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/CsvOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,14 @@

import com.google.common.base.MoreObjects;

import java.io.Serializable;
import java.nio.charset.Charset;
import java.util.Objects;

/**
* Google BigQuery CSV options. This class wraps some properties of CSV files used by BigQuery to
* parse external data.
* Google BigQuery options for CSV format. This class wraps some properties of CSV files used by
* BigQuery to parse external data.
*/
public class CsvOptions implements Serializable {
public class CsvOptions extends FormatOptions {

private static final long serialVersionUID = 2193570529308612708L;

Expand Down Expand Up @@ -132,6 +131,7 @@ public CsvOptions build() {
}

private CsvOptions(Builder builder) {
super(FormatOptions.CSV);
this.allowJaggedRows = builder.allowJaggedRows;
this.allowQuotedNewLines = builder.allowQuotedNewLines;
this.encoding = builder.encoding;
Expand Down Expand Up @@ -226,24 +226,12 @@ public boolean equals(Object obj) {
com.google.api.services.bigquery.model.CsvOptions toPb() {
com.google.api.services.bigquery.model.CsvOptions csvOptions =
new com.google.api.services.bigquery.model.CsvOptions();
if (allowJaggedRows != null) {
csvOptions.setAllowJaggedRows(allowJaggedRows);
}
if (allowQuotedNewLines != null) {
csvOptions.setAllowQuotedNewlines(allowQuotedNewLines);
}
if (encoding != null) {
csvOptions.setEncoding(encoding);
}
if (fieldDelimiter != null) {
csvOptions.setFieldDelimiter(fieldDelimiter);
}
if (quote != null) {
csvOptions.setQuote(quote);
}
if (skipLeadingRows != null) {
csvOptions.setSkipLeadingRows(skipLeadingRows);
}
csvOptions.setAllowJaggedRows(allowJaggedRows);
csvOptions.setAllowQuotedNewlines(allowQuotedNewLines);
csvOptions.setEncoding(encoding);
csvOptions.setFieldDelimiter(fieldDelimiter);
csvOptions.setQuote(quote);
csvOptions.setSkipLeadingRows(skipLeadingRows);
return csvOptions;
}

Expand Down
95 changes: 40 additions & 55 deletions gcloud-java-bigquery/src/main/java/com/google/gcloud/bigquery/ExternalDataConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,29 +63,27 @@ public com.google.api.services.bigquery.model.ExternalDataConfiguration apply(

private final List<String> sourceUris;
private final Schema schema;
private final String sourceFormat;
private final FormatOptions formatOptions;
private final Integer maxBadRecords;
private final Boolean ignoreUnknownValues;
private final String compression;
private final CsvOptions csvOptions;

public static final class Builder {

private List<String> sourceUris;
private Schema schema;
private String sourceFormat;
private FormatOptions formatOptions;
private Integer maxBadRecords;
private Boolean ignoreUnknownValues;
private String compression;
private CsvOptions csvOptions;

private Builder() {}

/**
* Sets the fully-qualified URIs that point to your data in Google Cloud Storage. Each URI can
* contain one '*' wildcard character that must come after the bucket's name. Size limits
* related to load jobs apply to external data sources, plus an additional limit of 10 GB
* maximum size across all URIs.
* Sets the fully-qualified URIs that point to your data in Google Cloud Storage (e.g.
* gs://bucket/path). Each URI can contain one '*' wildcard character that must come after the
* bucket's name. Size limits related to load jobs apply to external data sources, plus an
* additional limit of 10 GB maximum size across all URIs.
*
* @see <a href="https://cloud.google.com/bigquery/loading-data-into-bigquery#quota">Quota</a>
*/
Expand All @@ -103,15 +101,14 @@ public Builder schema(Schema schema) {
}

/**
* Sets the source format of the external data. Supported values are {@code CSV} for CSV files,
* and {@code NEWLINE_DELIMITED_JSON} for newline-delimited JSON. If not set, files are assumed
* to be in CSV format.
* Sets the source format, and possibly some parsing options, of the external data. Supported
* formats are {@code CSV} and {@code NEWLINE_DELIMITED_JSON}.
*
* <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public Builder sourceFormat(String sourceFormat) {
this.sourceFormat = checkNotNull(sourceFormat);
public Builder formatOptions(FormatOptions formatOptions) {
this.formatOptions = checkNotNull(formatOptions);
return this;
}

Expand All @@ -129,9 +126,8 @@ public Builder maxBadRecords(Integer maxBadRecords) {
* Sets whether BigQuery should allow extra values that are not represented in the table schema.
* If true, the extra values are ignored. If false, records with extra columns are treated as
* bad records, and if there are too many bad records, an invalid error is returned in the job
* result. The default value is false. The value set with
* {@link #sourceFormat(String)} property determines what
* BigQuery treats as an extra value.
* result. The default value is false. The value set with {@link #formatOptions(FormatOptions)}
* property determines what BigQuery treats as an extra value.
*
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.ignoreUnknownValues">
* Ignore Unknown Values</a>
Expand All @@ -152,15 +148,6 @@ public Builder compression(String compression) {
return this;
}

/**
* Sets additional properties to be used to parse CSV data (used when
* {@link #sourceFormat(String)} is set to CSV).
*/
public Builder csvOptions(CsvOptions csvOptions) {
this.csvOptions = csvOptions;
return this;
}

/**
* Creates an {@code ExternalDataConfiguration} object.
*/
Expand All @@ -174,9 +161,8 @@ public ExternalDataConfiguration build() {
this.ignoreUnknownValues = builder.ignoreUnknownValues;
this.maxBadRecords = builder.maxBadRecords;
this.schema = builder.schema;
this.sourceFormat = builder.sourceFormat;
this.formatOptions = builder.formatOptions;
this.sourceUris = builder.sourceUris;
this.csvOptions = builder.csvOptions;
}

/**
Expand All @@ -193,8 +179,8 @@ public String compression() {
* Returns whether BigQuery should allow extra values that are not represented in the table
* schema. If true, the extra values are ignored. If false, records with extra columns are treated
* as bad records, and if there are too many bad records, an invalid error is returned in the job
* result. The default value is false. The value of {@link #sourceFormat()} determines what
* BigQuery treats as an extra value.
* result. The default value is false. The value of {@link #format()} determines what BigQuery
* treats as an extra value.
*
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.ignoreUnknownValues">
* Ignore Unknown Values</a>
Expand All @@ -219,13 +205,13 @@ public Schema schema() {
}

/**
* Sets the source format of the external data.
* Returns the source format of the external data.
*
* <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public String sourceFormat() {
return sourceFormat;
public String format() {

This comment has been minimized.

Sign in to view

Copy link
@aozarov

aozarov Dec 3, 2015

Contributor

did you consider this:

@SuppressWarnings("unchecked")
public <F extends FormatOptions> F formatOptions() {
  return (F) formatOptions;
}

I think this could come instead of format() and csvOptions as "format" can always be taken by formatOptions().type() and one could either assign directly to CsvOptions if type is known or
assign to FormatOptions and cast based on type if needed.
return formatOptions.type();
}

/**
Expand All @@ -241,11 +227,11 @@ public List<String> sourceUris() {
}

/**
* Returns additional properties used to parse CSV data (used when {@link #sourceFormat()} is set
* to CSV).
* Returns additional properties used to parse CSV data (used when {@link #format()} is set to
* CSV). Returns {@code null} if not set.
*/
public CsvOptions csvOptions() {
return csvOptions;
return formatOptions instanceof CsvOptions ? (CsvOptions) formatOptions : null;
}

/**
Expand All @@ -257,28 +243,26 @@ public Builder toBuilder() {
.ignoreUnknownValues(ignoreUnknownValues)
.maxBadRecords(maxBadRecords)
.schema(schema)
.sourceFormat(sourceFormat)
.sourceUris(sourceUris)
.csvOptions(csvOptions);
.formatOptions(formatOptions)
.sourceUris(sourceUris);
}

@Override
public String toString() {
return MoreObjects.toStringHelper(this)
.add("sourceUris", sourceUris)
.add("sourceFormat", sourceFormat)
.add("formatOptions", formatOptions)
.add("schema", schema)
.add("compression", compression)
.add("ignoreUnknownValues", ignoreUnknownValues)
.add("maxBadRecords", maxBadRecords)
.add("csvOptions", csvOptions)
.toString();
}

@Override
public int hashCode() {
return Objects.hash(compression, ignoreUnknownValues, maxBadRecords, schema, sourceFormat,
sourceUris, csvOptions);
return Objects.hash(compression, ignoreUnknownValues, maxBadRecords, schema, formatOptions,
sourceUris);
}

@Override
Expand All @@ -302,14 +286,14 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() {
if (schema != null) {
externalConfigurationPb.setSchema(schema.toPb());
}
if (sourceFormat != null) {
externalConfigurationPb.setSourceFormat(sourceFormat);
if (formatOptions != null) {
externalConfigurationPb.setSourceFormat(formatOptions.type());
}
if (sourceUris != null) {
externalConfigurationPb.setSourceUris(sourceUris);
}
if (csvOptions != null) {
externalConfigurationPb.setCsvOptions(csvOptions.toPb());
if (csvOptions() != null) {
externalConfigurationPb.setCsvOptions(csvOptions().toPb());
}
return externalConfigurationPb;
}
Expand All @@ -329,8 +313,8 @@ com.google.api.services.bigquery.model.ExternalDataConfiguration toPb() {
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static Builder builder(List<String> sourceUris, Schema schema, String format) {
return new Builder().sourceUris(sourceUris).schema(schema).sourceFormat(format);
public static Builder builder(List<String> sourceUris, Schema schema, FormatOptions format) {
return new Builder().sourceUris(sourceUris).schema(schema).formatOptions(format);
}

/**
Expand All @@ -347,11 +331,11 @@ public static Builder builder(List<String> sourceUris, Schema schema, String for
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static Builder builder(String sourceUri, Schema schema, String format) {
public static Builder builder(String sourceUri, Schema schema, FormatOptions format) {
return new Builder()
.sourceUris(ImmutableList.of(sourceUri))
.schema(schema)
.sourceFormat(format);
.formatOptions(format);
}

/**
Expand All @@ -369,8 +353,8 @@ public static Builder builder(String sourceUri, Schema schema, String format) {
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static ExternalDataConfiguration of(
List<String> sourceUris, Schema schema, String format) {
public static ExternalDataConfiguration of(List<String> sourceUris, Schema schema,
FormatOptions format) {
return builder(sourceUris, schema, format).build();
}

Expand All @@ -388,7 +372,8 @@ public static ExternalDataConfiguration of(
* @see <a href="https://cloud.google.com/bigquery/docs/reference/v2/tables#externalDataConfiguration.sourceFormat">
* Source Format</a>
*/
public static ExternalDataConfiguration of(String sourceUri, Schema schema, String format) {
public static ExternalDataConfiguration of(String sourceUri, Schema schema,
FormatOptions format) {
return builder(sourceUri, schema, format).build();
}

Expand All @@ -402,7 +387,7 @@ static ExternalDataConfiguration fromPb(
builder.schema(Schema.fromPb(externalDataConfiguration.getSchema()));
}
if (externalDataConfiguration.getSourceFormat() != null) {
builder.sourceFormat(externalDataConfiguration.getSourceFormat());
builder.formatOptions(FormatOptions.of(externalDataConfiguration.getSourceFormat()));
}
if (externalDataConfiguration.getCompression() != null) {
builder.compression(externalDataConfiguration.getCompression());
Expand All @@ -411,7 +396,7 @@ static ExternalDataConfiguration fromPb(
builder.ignoreUnknownValues(externalDataConfiguration.getIgnoreUnknownValues());
}
if (externalDataConfiguration.getCsvOptions() != null) {
builder.csvOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions()));
builder.formatOptions(CsvOptions.fromPb(externalDataConfiguration.getCsvOptions()));
}
if (externalDataConfiguration.getMaxBadRecords() != null) {
builder.maxBadRecords(externalDataConfiguration.getMaxBadRecords());
Expand Down
Loading

0 comments on commit 79001cf

Please sign in to comment.