From 9bc3782b40ab81b06861c9af2f538cd86bba7604 Mon Sep 17 00:00:00 2001 From: Yoshi Automation Bot Date: Wed, 16 Sep 2020 07:58:34 -0700 Subject: [PATCH] feat: added support for per type and partition export for Cloud Asset API (#398) Clients can now specify two more args when export assets to bigquery PiperOrigin-RevId: 331912851 Source-Author: Google APIs Source-Date: Tue Sep 15 20:04:02 2020 -0700 Source-Repo: googleapis/googleapis Source-Sha: 5e53d6b6dde0e72fa9510ec1d796176d128afa40 Source-Link: https://github.com/googleapis/googleapis/commit/5e53d6b6dde0e72fa9510ec1d796176d128afa40 --- .../google/cloud/asset/v1/asset_service.proto | 77 +++++ .../google-cloud-asset/protos/protos.d.ts | 112 ++++++++ packages/google-cloud-asset/protos/protos.js | 270 ++++++++++++++++++ .../google-cloud-asset/protos/protos.json | 25 ++ packages/google-cloud-asset/synth.metadata | 6 +- 5 files changed, 487 insertions(+), 3 deletions(-) diff --git a/packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto b/packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto index 4aa3ee74ee2..6b66d4444dc 100644 --- a/packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto +++ b/packages/google-cloud-asset/protos/google/cloud/asset/v1/asset_service.proto @@ -393,6 +393,83 @@ message BigQueryDestination { // is `FALSE` or unset and the destination table already exists, the export // call returns an INVALID_ARGUMEMT error. bool force = 3; + + // [partition_spec] determines whether to export to partitioned table(s) and + // how to partition the data. + // + // If [partition_spec] is unset or [partition_spec.partion_key] is unset or + // `PARTITION_KEY_UNSPECIFIED`, the snapshot results will be exported to + // non-partitioned table(s). [force] will decide whether to overwrite existing + // table(s). + // + // If [partition_spec] is specified. First, the snapshot results will be + // written to partitioned table(s) with two additional timestamp columns, + // readTime and requestTime, one of which will be the partition key. Secondly, + // in the case when any destination table already exists, it will first try to + // update existing table's schema as necessary by appending additional + // columns. Then, if [force] is `TRUE`, the corresponding partition will be + // overwritten by the snapshot results (data in different partitions will + // remain intact); if [force] is unset or `FALSE`, it will append the data. An + // error will be returned if the schema update or data appension fails. + PartitionSpec partition_spec = 4; + + // If this flag is `TRUE`, the snapshot results will be written to one or + // multiple tables, each of which contains results of one asset type. The + // [force] and [partition_spec] fields will apply to each of them. + // + // Field [table] will be concatenated with "_" and the asset type names (see + // https://cloud.google.com/asset-inventory/docs/supported-asset-types for + // supported asset types) to construct per-asset-type table names, in which + // all non-alphanumeric characters like "." and "/" will be substituted by + // "_". Example: if field [table] is "mytable" and snapshot results + // contain "storage.googleapis.com/Bucket" assets, the corresponding table + // name will be "mytable_storage_googleapis_com_Bucket". If any of these + // tables does not exist, a new table with the concatenated name will be + // created. + // + // When [content_type] in the ExportAssetsRequest is `RESOURCE`, the schema of + // each table will include RECORD-type columns mapped to the nested fields in + // the Asset.resource.data field of that asset type (up to the 15 nested level + // BigQuery supports + // (https://cloud.google.com/bigquery/docs/nested-repeated#limitations)). The + // fields in >15 nested levels will be stored in JSON format string as a child + // column of its parent RECORD column. + // + // If error occurs when exporting to any table, the whole export call will + // return an error but the export results that already succeed will persist. + // Example: if exporting to table_type_A succeeds when exporting to + // table_type_B fails during one export call, the results in table_type_A will + // persist and there will not be partial results persisting in a table. + bool separate_tables_per_asset_type = 5; +} + +// Specifications of BigQuery partitioned table as export destination. +message PartitionSpec { + // This enum is used to determine the partition key column when exporting + // assets to BigQuery partitioned table(s). Note that, if the partition key is + // a timestamp column, the actual partition is based on its date value + // (expressed in UTC. see details in + // https://cloud.google.com/bigquery/docs/partitioned-tables#date_timestamp_partitioned_tables). + enum PartitionKey { + // Unspecified partition key. If used, it means using non-partitioned table. + PARTITION_KEY_UNSPECIFIED = 0; + + // The time when the snapshot is taken. If specified as partition key, the + // result table(s) is partitoned by the additional timestamp column, + // readTime. If [read_time] in ExportAssetsRequest is specified, the + // readTime column's value will be the same as it. Otherwise, its value will + // be the current time that is used to take the snapshot. + READ_TIME = 1; + + // The time when the request is received and started to be processed. If + // specified as partition key, the result table(s) is partitoned by the + // requestTime column, an additional timestamp column representing when the + // request was received. + REQUEST_TIME = 2; + } + + // The partition key for BigQuery partitioned table. + PartitionKey partition_key = 1; } // A Pub/Sub destination. diff --git a/packages/google-cloud-asset/protos/protos.d.ts b/packages/google-cloud-asset/protos/protos.d.ts index 5e5c844f96b..66662cd6888 100644 --- a/packages/google-cloud-asset/protos/protos.d.ts +++ b/packages/google-cloud-asset/protos/protos.d.ts @@ -1603,6 +1603,12 @@ export namespace google { /** BigQueryDestination force */ force?: (boolean|null); + + /** BigQueryDestination partitionSpec */ + partitionSpec?: (google.cloud.asset.v1.IPartitionSpec|null); + + /** BigQueryDestination separateTablesPerAssetType */ + separateTablesPerAssetType?: (boolean|null); } /** Represents a BigQueryDestination. */ @@ -1623,6 +1629,12 @@ export namespace google { /** BigQueryDestination force. */ public force: boolean; + /** BigQueryDestination partitionSpec. */ + public partitionSpec?: (google.cloud.asset.v1.IPartitionSpec|null); + + /** BigQueryDestination separateTablesPerAssetType. */ + public separateTablesPerAssetType: boolean; + /** * Creates a new BigQueryDestination instance using the specified properties. * @param [properties] Properties to set @@ -1694,6 +1706,106 @@ export namespace google { public toJSON(): { [k: string]: any }; } + /** Properties of a PartitionSpec. */ + interface IPartitionSpec { + + /** PartitionSpec partitionKey */ + partitionKey?: (google.cloud.asset.v1.PartitionSpec.PartitionKey|keyof typeof google.cloud.asset.v1.PartitionSpec.PartitionKey|null); + } + + /** Represents a PartitionSpec. */ + class PartitionSpec implements IPartitionSpec { + + /** + * Constructs a new PartitionSpec. + * @param [properties] Properties to set + */ + constructor(properties?: google.cloud.asset.v1.IPartitionSpec); + + /** PartitionSpec partitionKey. */ + public partitionKey: (google.cloud.asset.v1.PartitionSpec.PartitionKey|keyof typeof google.cloud.asset.v1.PartitionSpec.PartitionKey); + + /** + * Creates a new PartitionSpec instance using the specified properties. + * @param [properties] Properties to set + * @returns PartitionSpec instance + */ + public static create(properties?: google.cloud.asset.v1.IPartitionSpec): google.cloud.asset.v1.PartitionSpec; + + /** + * Encodes the specified PartitionSpec message. Does not implicitly {@link google.cloud.asset.v1.PartitionSpec.verify|verify} messages. + * @param message PartitionSpec message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encode(message: google.cloud.asset.v1.IPartitionSpec, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Encodes the specified PartitionSpec message, length delimited. Does not implicitly {@link google.cloud.asset.v1.PartitionSpec.verify|verify} messages. + * @param message PartitionSpec message or plain object to encode + * @param [writer] Writer to encode to + * @returns Writer + */ + public static encodeDelimited(message: google.cloud.asset.v1.IPartitionSpec, writer?: $protobuf.Writer): $protobuf.Writer; + + /** + * Decodes a PartitionSpec message from the specified reader or buffer. + * @param reader Reader or buffer to decode from + * @param [length] Message length if known beforehand + * @returns PartitionSpec + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decode(reader: ($protobuf.Reader|Uint8Array), length?: number): google.cloud.asset.v1.PartitionSpec; + + /** + * Decodes a PartitionSpec message from the specified reader or buffer, length delimited. + * @param reader Reader or buffer to decode from + * @returns PartitionSpec + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + public static decodeDelimited(reader: ($protobuf.Reader|Uint8Array)): google.cloud.asset.v1.PartitionSpec; + + /** + * Verifies a PartitionSpec message. + * @param message Plain object to verify + * @returns `null` if valid, otherwise the reason why it is not + */ + public static verify(message: { [k: string]: any }): (string|null); + + /** + * Creates a PartitionSpec message from a plain object. Also converts values to their respective internal types. + * @param object Plain object + * @returns PartitionSpec + */ + public static fromObject(object: { [k: string]: any }): google.cloud.asset.v1.PartitionSpec; + + /** + * Creates a plain object from a PartitionSpec message. Also converts values to other types if specified. + * @param message PartitionSpec + * @param [options] Conversion options + * @returns Plain object + */ + public static toObject(message: google.cloud.asset.v1.PartitionSpec, options?: $protobuf.IConversionOptions): { [k: string]: any }; + + /** + * Converts this PartitionSpec to JSON. + * @returns JSON object + */ + public toJSON(): { [k: string]: any }; + } + + namespace PartitionSpec { + + /** PartitionKey enum. */ + enum PartitionKey { + PARTITION_KEY_UNSPECIFIED = 0, + READ_TIME = 1, + REQUEST_TIME = 2 + } + } + /** Properties of a PubsubDestination. */ interface IPubsubDestination { diff --git a/packages/google-cloud-asset/protos/protos.js b/packages/google-cloud-asset/protos/protos.js index 75251f38f6e..525014aec75 100644 --- a/packages/google-cloud-asset/protos/protos.js +++ b/packages/google-cloud-asset/protos/protos.js @@ -3615,6 +3615,8 @@ * @property {string|null} [dataset] BigQueryDestination dataset * @property {string|null} [table] BigQueryDestination table * @property {boolean|null} [force] BigQueryDestination force + * @property {google.cloud.asset.v1.IPartitionSpec|null} [partitionSpec] BigQueryDestination partitionSpec + * @property {boolean|null} [separateTablesPerAssetType] BigQueryDestination separateTablesPerAssetType */ /** @@ -3656,6 +3658,22 @@ */ BigQueryDestination.prototype.force = false; + /** + * BigQueryDestination partitionSpec. + * @member {google.cloud.asset.v1.IPartitionSpec|null|undefined} partitionSpec + * @memberof google.cloud.asset.v1.BigQueryDestination + * @instance + */ + BigQueryDestination.prototype.partitionSpec = null; + + /** + * BigQueryDestination separateTablesPerAssetType. + * @member {boolean} separateTablesPerAssetType + * @memberof google.cloud.asset.v1.BigQueryDestination + * @instance + */ + BigQueryDestination.prototype.separateTablesPerAssetType = false; + /** * Creates a new BigQueryDestination instance using the specified properties. * @function create @@ -3686,6 +3704,10 @@ writer.uint32(/* id 2, wireType 2 =*/18).string(message.table); if (message.force != null && Object.hasOwnProperty.call(message, "force")) writer.uint32(/* id 3, wireType 0 =*/24).bool(message.force); + if (message.partitionSpec != null && Object.hasOwnProperty.call(message, "partitionSpec")) + $root.google.cloud.asset.v1.PartitionSpec.encode(message.partitionSpec, writer.uint32(/* id 4, wireType 2 =*/34).fork()).ldelim(); + if (message.separateTablesPerAssetType != null && Object.hasOwnProperty.call(message, "separateTablesPerAssetType")) + writer.uint32(/* id 5, wireType 0 =*/40).bool(message.separateTablesPerAssetType); return writer; }; @@ -3729,6 +3751,12 @@ case 3: message.force = reader.bool(); break; + case 4: + message.partitionSpec = $root.google.cloud.asset.v1.PartitionSpec.decode(reader, reader.uint32()); + break; + case 5: + message.separateTablesPerAssetType = reader.bool(); + break; default: reader.skipType(tag & 7); break; @@ -3773,6 +3801,14 @@ if (message.force != null && message.hasOwnProperty("force")) if (typeof message.force !== "boolean") return "force: boolean expected"; + if (message.partitionSpec != null && message.hasOwnProperty("partitionSpec")) { + var error = $root.google.cloud.asset.v1.PartitionSpec.verify(message.partitionSpec); + if (error) + return "partitionSpec." + error; + } + if (message.separateTablesPerAssetType != null && message.hasOwnProperty("separateTablesPerAssetType")) + if (typeof message.separateTablesPerAssetType !== "boolean") + return "separateTablesPerAssetType: boolean expected"; return null; }; @@ -3794,6 +3830,13 @@ message.table = String(object.table); if (object.force != null) message.force = Boolean(object.force); + if (object.partitionSpec != null) { + if (typeof object.partitionSpec !== "object") + throw TypeError(".google.cloud.asset.v1.BigQueryDestination.partitionSpec: object expected"); + message.partitionSpec = $root.google.cloud.asset.v1.PartitionSpec.fromObject(object.partitionSpec); + } + if (object.separateTablesPerAssetType != null) + message.separateTablesPerAssetType = Boolean(object.separateTablesPerAssetType); return message; }; @@ -3814,6 +3857,8 @@ object.dataset = ""; object.table = ""; object.force = false; + object.partitionSpec = null; + object.separateTablesPerAssetType = false; } if (message.dataset != null && message.hasOwnProperty("dataset")) object.dataset = message.dataset; @@ -3821,6 +3866,10 @@ object.table = message.table; if (message.force != null && message.hasOwnProperty("force")) object.force = message.force; + if (message.partitionSpec != null && message.hasOwnProperty("partitionSpec")) + object.partitionSpec = $root.google.cloud.asset.v1.PartitionSpec.toObject(message.partitionSpec, options); + if (message.separateTablesPerAssetType != null && message.hasOwnProperty("separateTablesPerAssetType")) + object.separateTablesPerAssetType = message.separateTablesPerAssetType; return object; }; @@ -3838,6 +3887,227 @@ return BigQueryDestination; })(); + v1.PartitionSpec = (function() { + + /** + * Properties of a PartitionSpec. + * @memberof google.cloud.asset.v1 + * @interface IPartitionSpec + * @property {google.cloud.asset.v1.PartitionSpec.PartitionKey|null} [partitionKey] PartitionSpec partitionKey + */ + + /** + * Constructs a new PartitionSpec. + * @memberof google.cloud.asset.v1 + * @classdesc Represents a PartitionSpec. + * @implements IPartitionSpec + * @constructor + * @param {google.cloud.asset.v1.IPartitionSpec=} [properties] Properties to set + */ + function PartitionSpec(properties) { + if (properties) + for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) + if (properties[keys[i]] != null) + this[keys[i]] = properties[keys[i]]; + } + + /** + * PartitionSpec partitionKey. + * @member {google.cloud.asset.v1.PartitionSpec.PartitionKey} partitionKey + * @memberof google.cloud.asset.v1.PartitionSpec + * @instance + */ + PartitionSpec.prototype.partitionKey = 0; + + /** + * Creates a new PartitionSpec instance using the specified properties. + * @function create + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {google.cloud.asset.v1.IPartitionSpec=} [properties] Properties to set + * @returns {google.cloud.asset.v1.PartitionSpec} PartitionSpec instance + */ + PartitionSpec.create = function create(properties) { + return new PartitionSpec(properties); + }; + + /** + * Encodes the specified PartitionSpec message. Does not implicitly {@link google.cloud.asset.v1.PartitionSpec.verify|verify} messages. + * @function encode + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {google.cloud.asset.v1.IPartitionSpec} message PartitionSpec message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + PartitionSpec.encode = function encode(message, writer) { + if (!writer) + writer = $Writer.create(); + if (message.partitionKey != null && Object.hasOwnProperty.call(message, "partitionKey")) + writer.uint32(/* id 1, wireType 0 =*/8).int32(message.partitionKey); + return writer; + }; + + /** + * Encodes the specified PartitionSpec message, length delimited. Does not implicitly {@link google.cloud.asset.v1.PartitionSpec.verify|verify} messages. + * @function encodeDelimited + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {google.cloud.asset.v1.IPartitionSpec} message PartitionSpec message or plain object to encode + * @param {$protobuf.Writer} [writer] Writer to encode to + * @returns {$protobuf.Writer} Writer + */ + PartitionSpec.encodeDelimited = function encodeDelimited(message, writer) { + return this.encode(message, writer).ldelim(); + }; + + /** + * Decodes a PartitionSpec message from the specified reader or buffer. + * @function decode + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @param {number} [length] Message length if known beforehand + * @returns {google.cloud.asset.v1.PartitionSpec} PartitionSpec + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + PartitionSpec.decode = function decode(reader, length) { + if (!(reader instanceof $Reader)) + reader = $Reader.create(reader); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.google.cloud.asset.v1.PartitionSpec(); + while (reader.pos < end) { + var tag = reader.uint32(); + switch (tag >>> 3) { + case 1: + message.partitionKey = reader.int32(); + break; + default: + reader.skipType(tag & 7); + break; + } + } + return message; + }; + + /** + * Decodes a PartitionSpec message from the specified reader or buffer, length delimited. + * @function decodeDelimited + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {$protobuf.Reader|Uint8Array} reader Reader or buffer to decode from + * @returns {google.cloud.asset.v1.PartitionSpec} PartitionSpec + * @throws {Error} If the payload is not a reader or valid buffer + * @throws {$protobuf.util.ProtocolError} If required fields are missing + */ + PartitionSpec.decodeDelimited = function decodeDelimited(reader) { + if (!(reader instanceof $Reader)) + reader = new $Reader(reader); + return this.decode(reader, reader.uint32()); + }; + + /** + * Verifies a PartitionSpec message. + * @function verify + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {Object.} message Plain object to verify + * @returns {string|null} `null` if valid, otherwise the reason why it is not + */ + PartitionSpec.verify = function verify(message) { + if (typeof message !== "object" || message === null) + return "object expected"; + if (message.partitionKey != null && message.hasOwnProperty("partitionKey")) + switch (message.partitionKey) { + default: + return "partitionKey: enum value expected"; + case 0: + case 1: + case 2: + break; + } + return null; + }; + + /** + * Creates a PartitionSpec message from a plain object. Also converts values to their respective internal types. + * @function fromObject + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {Object.} object Plain object + * @returns {google.cloud.asset.v1.PartitionSpec} PartitionSpec + */ + PartitionSpec.fromObject = function fromObject(object) { + if (object instanceof $root.google.cloud.asset.v1.PartitionSpec) + return object; + var message = new $root.google.cloud.asset.v1.PartitionSpec(); + switch (object.partitionKey) { + case "PARTITION_KEY_UNSPECIFIED": + case 0: + message.partitionKey = 0; + break; + case "READ_TIME": + case 1: + message.partitionKey = 1; + break; + case "REQUEST_TIME": + case 2: + message.partitionKey = 2; + break; + } + return message; + }; + + /** + * Creates a plain object from a PartitionSpec message. Also converts values to other types if specified. + * @function toObject + * @memberof google.cloud.asset.v1.PartitionSpec + * @static + * @param {google.cloud.asset.v1.PartitionSpec} message PartitionSpec + * @param {$protobuf.IConversionOptions} [options] Conversion options + * @returns {Object.} Plain object + */ + PartitionSpec.toObject = function toObject(message, options) { + if (!options) + options = {}; + var object = {}; + if (options.defaults) + object.partitionKey = options.enums === String ? "PARTITION_KEY_UNSPECIFIED" : 0; + if (message.partitionKey != null && message.hasOwnProperty("partitionKey")) + object.partitionKey = options.enums === String ? $root.google.cloud.asset.v1.PartitionSpec.PartitionKey[message.partitionKey] : message.partitionKey; + return object; + }; + + /** + * Converts this PartitionSpec to JSON. + * @function toJSON + * @memberof google.cloud.asset.v1.PartitionSpec + * @instance + * @returns {Object.} JSON object + */ + PartitionSpec.prototype.toJSON = function toJSON() { + return this.constructor.toObject(this, $protobuf.util.toJSONOptions); + }; + + /** + * PartitionKey enum. + * @name google.cloud.asset.v1.PartitionSpec.PartitionKey + * @enum {number} + * @property {number} PARTITION_KEY_UNSPECIFIED=0 PARTITION_KEY_UNSPECIFIED value + * @property {number} READ_TIME=1 READ_TIME value + * @property {number} REQUEST_TIME=2 REQUEST_TIME value + */ + PartitionSpec.PartitionKey = (function() { + var valuesById = {}, values = Object.create(valuesById); + values[valuesById[0] = "PARTITION_KEY_UNSPECIFIED"] = 0; + values[valuesById[1] = "READ_TIME"] = 1; + values[valuesById[2] = "REQUEST_TIME"] = 2; + return values; + })(); + + return PartitionSpec; + })(); + v1.PubsubDestination = (function() { /** diff --git a/packages/google-cloud-asset/protos/protos.json b/packages/google-cloud-asset/protos/protos.json index e4d819f310f..d982a76543b 100644 --- a/packages/google-cloud-asset/protos/protos.json +++ b/packages/google-cloud-asset/protos/protos.json @@ -358,6 +358,31 @@ "force": { "type": "bool", "id": 3 + }, + "partitionSpec": { + "type": "PartitionSpec", + "id": 4 + }, + "separateTablesPerAssetType": { + "type": "bool", + "id": 5 + } + } + }, + "PartitionSpec": { + "fields": { + "partitionKey": { + "type": "PartitionKey", + "id": 1 + } + }, + "nested": { + "PartitionKey": { + "values": { + "PARTITION_KEY_UNSPECIFIED": 0, + "READ_TIME": 1, + "REQUEST_TIME": 2 + } } } }, diff --git a/packages/google-cloud-asset/synth.metadata b/packages/google-cloud-asset/synth.metadata index 96c9eb419fc..d11546680f1 100644 --- a/packages/google-cloud-asset/synth.metadata +++ b/packages/google-cloud-asset/synth.metadata @@ -4,15 +4,15 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/nodejs-asset.git", - "sha": "1d45e05b9241ba18fd906d5c9ac87653e1cc795d" + "sha": "1c1eb901a33c78ff2271651f53df165f6fab42b5" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "ef03f63f2f2d3b2dd936e46595c0f746cb10c43c", - "internalRef": "330786980" + "sha": "5e53d6b6dde0e72fa9510ec1d796176d128afa40", + "internalRef": "331912851" } }, {