feat: added support for per type and partition export for Cloud Asset…

… API (#398)

Clients can now specify two more args when export assets to bigquery

PiperOrigin-RevId: 331912851

Source-Author: Google APIs <noreply@google.com>
Source-Date: Tue Sep 15 20:04:02 2020 -0700
Source-Repo: googleapis/googleapis
Source-Sha: 5e53d6b6dde0e72fa9510ec1d796176d128afa40
Source-Link: googleapis/googleapis@5e53d6b

packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto

@@ -393,6 +393,83 @@ message BigQueryDestination {
  // is `FALSE` or unset and the destination table already exists, the export
  // call returns an INVALID_ARGUMEMT error.
  bool force = 3;
+
+ // [partition_spec] determines whether to export to partitioned table(s) and
+ // how to partition the data.
+ //
+ // If [partition_spec] is unset or [partition_spec.partion_key] is unset or
+ // `PARTITION_KEY_UNSPECIFIED`, the snapshot results will be exported to
+ // non-partitioned table(s). [force] will decide whether to overwrite existing
+ // table(s).
+ //
+ // If [partition_spec] is specified. First, the snapshot results will be
+ // written to partitioned table(s) with two additional timestamp columns,
+ // readTime and requestTime, one of which will be the partition key. Secondly,
+ // in the case when any destination table already exists, it will first try to
+ // update existing table's schema as necessary by appending additional
+ // columns. Then, if [force] is `TRUE`, the corresponding partition will be
+ // overwritten by the snapshot results (data in different partitions will
+ // remain intact); if [force] is unset or `FALSE`, it will append the data. An
+ // error will be returned if the schema update or data appension fails.
+ PartitionSpec partition_spec = 4;
+
+ // If this flag is `TRUE`, the snapshot results will be written to one or
+ // multiple tables, each of which contains results of one asset type. The
+ // [force] and [partition_spec] fields will apply to each of them.
+ //
+ // Field [table] will be concatenated with "_" and the asset type names (see
+ // https://cloud.google.com/asset-inventory/docs/supported-asset-types for
+ // supported asset types) to construct per-asset-type table names, in which
+ // all non-alphanumeric characters like "." and "/" will be substituted by
+ // "_". Example: if field [table] is "mytable" and snapshot results
+ // contain "storage.googleapis.com/Bucket" assets, the corresponding table
+ // name will be "mytable_storage_googleapis_com_Bucket". If any of these
+ // tables does not exist, a new table with the concatenated name will be
+ // created.
+ //
+ // When [content_type] in the ExportAssetsRequest is `RESOURCE`, the schema of
+ // each table will include RECORD-type columns mapped to the nested fields in
+ // the Asset.resource.data field of that asset type (up to the 15 nested level
+ // BigQuery supports
+ // (https://cloud.google.com/bigquery/docs/nested-repeated#limitations)). The
+ // fields in >15 nested levels will be stored in JSON format string as a child
+ // column of its parent RECORD column.
+ //
+ // If error occurs when exporting to any table, the whole export call will
+ // return an error but the export results that already succeed will persist.
+ // Example: if exporting to table_type_A succeeds when exporting to
+ // table_type_B fails during one export call, the results in table_type_A will
+ // persist and there will not be partial results persisting in a table.
+ bool separate_tables_per_asset_type = 5;
+}
+
+// Specifications of BigQuery partitioned table as export destination.
+message PartitionSpec {
+ // This enum is used to determine the partition key column when exporting
+ // assets to BigQuery partitioned table(s). Note that, if the partition key is
+ // a timestamp column, the actual partition is based on its date value
+ // (expressed in UTC. see details in
+ // https://cloud.google.com/bigquery/docs/partitioned-tables#date_timestamp_partitioned_tables).
+ enum PartitionKey {
+ // Unspecified partition key. If used, it means using non-partitioned table.
+ PARTITION_KEY_UNSPECIFIED = 0;
+
+ // The time when the snapshot is taken. If specified as partition key, the
+ // result table(s) is partitoned by the additional timestamp column,
+ // readTime. If [read_time] in ExportAssetsRequest is specified, the
+ // readTime column's value will be the same as it. Otherwise, its value will
+ // be the current time that is used to take the snapshot.
+ READ_TIME = 1;
+
+ // The time when the request is received and started to be processed. If
+ // specified as partition key, the result table(s) is partitoned by the
+ // requestTime column, an additional timestamp column representing when the
+ // request was received.
+ REQUEST_TIME = 2;
+ }
+
+ // The partition key for BigQuery partitioned table.
+ PartitionKey partition_key = 1;
 }
 
 // A Pub/Sub destination.