Skip to content

Commit

Permalink
feat: add resource reference for KMS keys and fix comments
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 674408200
  • Loading branch information
@copybara-github
Google APIs authored and copybara-github committed Sep 13, 2024
1 parent ffbe783 commit 964f6c9
Show file tree
Hide file tree
Showing 4 changed files with 85 additions and 55 deletions.
66 changes: 42 additions & 24 deletions google/cloud/dataproc/v1/clusters.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -379,8 +379,11 @@ message AutoscalingConfig {

// Encryption settings for the cluster.
message EncryptionConfig {
// Optional. The Cloud KMS key name to use for PD disk encryption for all
// instances in the cluster.
// Optional. The Cloud KMS key resource name to use for persistent disk
// encryption for all instances in the cluster. See [Use CMEK with cluster
// data]
// (https://cloud.google.com//dataproc/docs/concepts/configuring-clusters/customer-managed-encryption#use_cmek_with_cluster_data)
// for more information.
string gce_pd_kms_key_name = 1 [(google.api.field_behavior) = OPTIONAL];

// Optional. The Cloud KMS key resource name to use for cluster persistent
Expand Down Expand Up @@ -409,7 +412,12 @@ message EncryptionConfig {
// scriptVariables and queryList.queries
// * [PrestoJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PrestoJob)
// scriptVariables and queryList.queries
string kms_key = 2 [(google.api.field_behavior) = OPTIONAL];
string kms_key = 2 [
(google.api.field_behavior) = OPTIONAL,
(google.api.resource_reference) = {
type: "cloudkms.googleapis.com/CryptoKey"
}
];
}

// Common config settings for resources of Compute Engine cluster
Expand Down Expand Up @@ -475,12 +483,22 @@ message GceClusterConfig {
// * `sub0`
string subnetwork_uri = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. If true, all instances in the cluster will only have internal IP
// addresses. By default, clusters are not restricted to internal IP
// addresses, and will have ephemeral external IP addresses assigned to each
// instance. This `internal_ip_only` restriction can only be enabled for
// subnetwork enabled networks, and all off-cluster dependencies must be
// configured to be accessible without external IP addresses.
// Optional. This setting applies to subnetwork-enabled networks. It is set to
// `true` by default in clusters created with image versions 2.2.x.
//
// When set to `true`:
//
// * All cluster VMs have internal IP addresses.
// * [Google Private Access]
// (https://cloud.google.com/vpc/docs/private-google-access)
// must be enabled to access Dataproc and other Google Cloud APIs.
// * Off-cluster dependencies must be configured to be accessible
// without external IP addresses.
//
// When set to `false`:
//
// * Cluster VMs are not restricted to internal IP addresses.
// * Ephemeral external IP addresses are assigned to each cluster VM.
optional bool internal_ip_only = 7 [(google.api.field_behavior) = OPTIONAL];

// Optional. The type of IPv6 access for a cluster.
Expand Down Expand Up @@ -517,8 +535,8 @@ message GceClusterConfig {
repeated string service_account_scopes = 3
[(google.api.field_behavior) = OPTIONAL];

// The Compute Engine tags to add to all instances (see [Tagging
// instances](https://cloud.google.com/compute/docs/label-or-tag-resources#tags)).
// The Compute Engine network tags to add to all instances (see [Tagging
// instances](https://cloud.google.com/vpc/docs/add-remove-network-tags)).
repeated string tags = 4;

// Optional. The Compute Engine metadata entries to add to all instances (see
Expand Down Expand Up @@ -821,15 +839,15 @@ message AcceleratorConfig {
//
// Examples:
//
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-k80`
// * `projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-k80`
// * `nvidia-tesla-k80`
// * `https://www.googleapis.com/compute/v1/projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-t4`
// * `projects/[project_id]/zones/[zone]/acceleratorTypes/nvidia-tesla-t4`
// * `nvidia-tesla-t4`
//
// **Auto Zone Exception**: If you are using the Dataproc
// [Auto Zone
// Placement](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/auto-zone#using_auto_zone_placement)
// feature, you must use the short name of the accelerator type
// resource, for example, `nvidia-tesla-k80`.
// resource, for example, `nvidia-tesla-t4`.
string accelerator_type_uri = 1;

// The number of the accelerator cards of this type exposed to this instance.
Expand Down Expand Up @@ -1039,7 +1057,7 @@ message KerberosConfig {
string root_principal_password_uri = 2
[(google.api.field_behavior) = OPTIONAL];

// Optional. The uri of the KMS key used to encrypt various sensitive
// Optional. The URI of the KMS key used to encrypt sensitive
// files.
string kms_key_uri = 3 [(google.api.field_behavior) = OPTIONAL];

Expand Down Expand Up @@ -1113,7 +1131,7 @@ message IdentityConfig {
message SoftwareConfig {
// Optional. The version of software inside the cluster. It must be one of the
// supported [Dataproc
// Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported_dataproc_versions),
// Versions](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#supported-dataproc-image-versions),
// such as "1.2" (including a subminor version, such as "1.2.29"), or the
// ["preview"
// version](https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-versions#other_versions).
Expand Down Expand Up @@ -1539,12 +1557,12 @@ message ListClustersRequest {
// where **field** is one of `status.state`, `clusterName`, or `labels.[KEY]`,
// and `[KEY]` is a label key. **value** can be `*` to match all values.
// `status.state` can be one of the following: `ACTIVE`, `INACTIVE`,
// `CREATING`, `RUNNING`, `ERROR`, `DELETING`, or `UPDATING`. `ACTIVE`
// contains the `CREATING`, `UPDATING`, and `RUNNING` states. `INACTIVE`
// contains the `DELETING` and `ERROR` states.
// `clusterName` is the name of the cluster provided at creation time.
// Only the logical `AND` operator is supported; space-separated items are
// treated as having an implicit `AND` operator.
// `CREATING`, `RUNNING`, `ERROR`, `DELETING`, `UPDATING`, `STOPPING`, or
// `STOPPED`. `ACTIVE` contains the `CREATING`, `UPDATING`, and `RUNNING`
// states. `INACTIVE` contains the `DELETING`, `ERROR`, `STOPPING`, and
// `STOPPED` states. `clusterName` is the name of the cluster provided at
// creation time. Only the logical `AND` operator is supported;
// space-separated items are treated as having an implicit `AND` operator.
//
// Example filter:
//
Expand Down Expand Up @@ -1596,7 +1614,7 @@ message DiagnoseClusterRequest {
// Required. The cluster name.
string cluster_name = 2 [(google.api.field_behavior) = REQUIRED];

// Optional. The output Cloud Storage directory for the diagnostic
// Optional. (Optional) The output Cloud Storage directory for the diagnostic
// tarball. If not specified, a task-specific directory in the cluster's
// staging bucket will be used.
string tarball_gcs_dir = 4 [(google.api.field_behavior) = OPTIONAL];
Expand Down
54 changes: 28 additions & 26 deletions google/cloud/dataproc/v1/jobs.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ message LoggingConfig {
OFF = 8;
}

// The per-package log levels for the driver. This may include
// The per-package log levels for the driver. This can include
// "root" package name to configure rootLogger.
// Examples:
// - 'com.google = FATAL'
Expand Down Expand Up @@ -174,7 +174,7 @@ message HadoopJob {

// Optional. The arguments to pass to the driver. Do not
// include arguments, such as `-libjars` or `-Dfoo=bar`, that can be set as
// job properties, since a collision may occur that causes an incorrect job
// job properties, since a collision might occur that causes an incorrect job
// submission.
repeated string args = 3 [(google.api.field_behavior) = OPTIONAL];

Expand All @@ -193,7 +193,7 @@ message HadoopJob {
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names to values, used to configure Hadoop.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in `/etc/hadoop/conf/*-site` and
// classes in user code.
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
Expand All @@ -208,14 +208,16 @@ message SparkJob {
// Required. The specification of the main method to call to drive the job.
// Specify either the jar file that contains the main class or the main class
// name. To pass both a main jar and a main class in that jar, add the jar to
// `CommonJob.jar_file_uris`, and then specify the main class name in
// `main_class`.
// [jarFileUris][google.cloud.dataproc.v1.SparkJob.jar_file_uris], and then
// specify the main class name in
// [mainClass][google.cloud.dataproc.v1.SparkJob.main_class].
oneof driver {
// The HCFS URI of the jar file that contains the main class.
string main_jar_file_uri = 1;

// The name of the driver's main class. The jar file that contains the class
// must be in the default CLASSPATH or specified in `jar_file_uris`.
// must be in the default CLASSPATH or specified in
// SparkJob.jar_file_uris.
string main_class = 2;
}

Expand All @@ -238,7 +240,7 @@ message SparkJob {
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names to values, used to configure Spark.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in
// /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -279,7 +281,7 @@ message PySparkJob {
repeated string archive_uris = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names to values, used to configure PySpark.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in
// /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -331,7 +333,7 @@ message HiveJob {
[(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names and values, used to configure Hive.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in `/etc/hadoop/conf/*-site.xml`,
// /etc/hive/conf/hive-site.xml, and classes in user code.
map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -362,7 +364,7 @@ message SparkSqlJob {

// Optional. A mapping of property names to values, used to configure
// Spark SQL's SparkConf. Properties that conflict with values set by the
// Dataproc API may be overwritten.
// Dataproc API might be overwritten.
map<string, string> properties = 4 [(google.api.field_behavior) = OPTIONAL];

// Optional. HCFS URIs of jar files to be added to the Spark CLASSPATH.
Expand Down Expand Up @@ -396,7 +398,7 @@ message PigJob {
[(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names to values, used to configure Pig.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in `/etc/hadoop/conf/*-site.xml`,
// /etc/pig/conf/pig.properties, and classes in user code.
map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -432,7 +434,7 @@ message SparkRJob {
repeated string archive_uris = 4 [(google.api.field_behavior) = OPTIONAL];

// Optional. A mapping of property names to values, used to configure SparkR.
// Properties that conflict with values set by the Dataproc API may be
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in
// /etc/spark/conf/spark-defaults.conf and classes in user code.
map<string, string> properties = 5 [(google.api.field_behavior) = OPTIONAL];
Expand Down Expand Up @@ -549,7 +551,7 @@ message FlinkJob {
// Optional. A mapping of property names to values, used to configure Flink.
// Properties that conflict with values set by the Dataproc API might be
// overwritten. Can include properties set in
// /etc/flink/conf/flink-defaults.conf and classes in user code.
// `/etc/flink/conf/flink-defaults.conf` and classes in user code.
map<string, string> properties = 7 [(google.api.field_behavior) = OPTIONAL];

// Optional. The runtime log config for job execution.
Expand Down Expand Up @@ -621,14 +623,14 @@ message JobStatus {
// Applies to RUNNING state.
SUBMITTED = 1;

// The Job has been received and is awaiting execution (it may be waiting
// The Job has been received and is awaiting execution (it might be waiting
// for a condition to be met). See the "details" field for the reason for
// the delay.
//
// Applies to RUNNING state.
QUEUED = 2;

// The agent-reported status is out of date, which may be caused by a
// The agent-reported status is out of date, which can be caused by a
// loss of communication between the agent and Dataproc. If the
// agent does not send a timely update, the job will fail.
//
Expand All @@ -640,7 +642,7 @@ message JobStatus {
State state = 1 [(google.api.field_behavior) = OUTPUT_ONLY];

// Optional. Output only. Job state details, such as an error
// description if the state is <code>ERROR</code>.
// description if the state is `ERROR`.
string details = 2 [
(google.api.field_behavior) = OUTPUT_ONLY,
(google.api.field_behavior) = OPTIONAL
Expand Down Expand Up @@ -769,7 +771,7 @@ message Job {
}

// Output only. The job status. Additional application-specific
// status information may be contained in the <code>type_job</code>
// status information might be contained in the <code>type_job</code>
// and <code>yarn_applications</code> fields.
JobStatus status = 8 [(google.api.field_behavior) = OUTPUT_ONLY];

Expand All @@ -780,7 +782,7 @@ message Job {
// Output only. The collection of YARN applications spun up by this job.
//
// **Beta** Feature: This report is available for testing purposes only. It
// may be changed before final release.
// might be changed before final release.
repeated YarnApplication yarn_applications = 9
[(google.api.field_behavior) = OUTPUT_ONLY];

Expand All @@ -790,15 +792,15 @@ message Job {
[(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. If present, the location of miscellaneous control files
// which may be used as part of job setup and handling. If not present,
// control files may be placed in the same location as `driver_output_uri`.
// which can be used as part of job setup and handling. If not present,
// control files might be placed in the same location as `driver_output_uri`.
string driver_control_files_uri = 15
[(google.api.field_behavior) = OUTPUT_ONLY];

// Optional. The labels to associate with this job.
// Label **keys** must contain 1 to 63 characters, and must conform to
// [RFC 1035](https://www.ietf.org/rfc/rfc1035.txt).
// Label **values** may be empty, but, if present, must contain 1 to 63
// Label **values** can be empty, but, if present, must contain 1 to 63
// characters, and must conform to [RFC
// 1035](https://www.ietf.org/rfc/rfc1035.txt). No more than 32 labels can be
// associated with a job.
Expand All @@ -809,7 +811,7 @@ message Job {

// Output only. A UUID that uniquely identifies a job within the project
// over time. This is in contrast to a user-settable reference.job_id that
// may be reused over time.
// might be reused over time.
string job_uuid = 22 [(google.api.field_behavior) = OUTPUT_ONLY];

// Output only. Indicates whether the job is completed. If the value is
Expand All @@ -834,12 +836,12 @@ message DriverSchedulingConfig {

// Job scheduling options.
message JobScheduling {
// Optional. Maximum number of times per hour a driver may be restarted as
// Optional. Maximum number of times per hour a driver can be restarted as
// a result of driver exiting with non-zero code before job is
// reported failed.
//
// A job may be reported as thrashing if the driver exits with a non-zero code
// four times within a 10-minute window.
// A job might be reported as thrashing if the driver exits with a non-zero
// code four times within a 10-minute window.
//
// Maximum value is 10.
//
Expand All @@ -848,7 +850,7 @@ message JobScheduling {
// (https://cloud.google.com/dataproc/docs/concepts/workflows/using-workflows#adding_jobs_to_a_template).
int32 max_failures_per_hour = 1 [(google.api.field_behavior) = OPTIONAL];

// Optional. Maximum total number of times a driver may be restarted as a
// Optional. Maximum total number of times a driver can be restarted as a
// result of the driver exiting with a non-zero code. After the maximum number
// is reached, the job will be reported as failed.
//
Expand Down
13 changes: 9 additions & 4 deletions google/cloud/dataproc/v1/shared.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ option (google.api.resource_definition) = {
type: "metastore.googleapis.com/Service"
pattern: "projects/{project}/locations/{location}/services/{service}"
};
option (google.api.resource_definition) = {
type: "cloudkms.googleapis.com/CryptoKey"
pattern: "projects/{project}/locations/{location}/keyRings/{key_ring}/cryptoKeys/{crypto_key}"
};

// Runtime configuration for a workload.
message RuntimeConfig {
Expand Down Expand Up @@ -512,10 +516,11 @@ enum Component {
// Unspecified component. Specifying this will cause Cluster creation to fail.
COMPONENT_UNSPECIFIED = 0;

// The Anaconda python distribution. The Anaconda component is not supported
// in the Dataproc [2.0 image]
// (/https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-release-2.0).
// The 2.0 image is pre-installed with Miniconda.
// The Anaconda component is no longer supported or applicable to
// [supported Dataproc on Compute Engine image versions]
// (https://cloud.google.com/dataproc/docs/concepts/versioning/dataproc-version-clusters#supported-dataproc-image-versions).
// It cannot be activated on clusters created with supported Dataproc on
// Compute Engine image versions.
ANACONDA = 5;

// Docker
Expand Down
7 changes: 6 additions & 1 deletion google/cloud/dataproc/v1/workflow_templates.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,12 @@ message WorkflowTemplate {
// scriptVariables and queryList.queries
// * [PrestoJob](https://cloud.google.com/dataproc/docs/reference/rest/v1/PrestoJob)
// scriptVariables and queryList.queries
string kms_key = 1 [(google.api.field_behavior) = OPTIONAL];
string kms_key = 1 [
(google.api.field_behavior) = OPTIONAL,
(google.api.resource_reference) = {
type: "cloudkms.googleapis.com/CryptoKey"
}
];
}

string id = 2 [(google.api.field_behavior) = REQUIRED];
Expand Down

0 comments on commit 964f6c9

Please sign in to comment.