Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support required_registration_fraction for secondary workers #11970

Merged
merged 2 commits into from
Nov 2, 2023
Merged

feat: support required_registration_fraction for secondary workers #11970

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
19a3d53
feat: support required_registration_fraction for secondary workers
gcf-owl-bot[bot] Nov 2, 2023
b078810
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] Nov 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions packages/google-cloud-dataproc/google/cloud/dataproc/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@
ShieldedInstanceConfig,
SoftwareConfig,
StartClusterRequest,
StartupConfig,
StopClusterRequest,
UpdateClusterRequest,
VirtualClusterConfig,
Expand Down Expand Up @@ -268,6 +269,7 @@
"ShieldedInstanceConfig",
"SoftwareConfig",
"StartClusterRequest",
"StartupConfig",
"StopClusterRequest",
"UpdateClusterRequest",
"VirtualClusterConfig",
Expand Down
2 changes: 2 additions & 0 deletions packages/google-cloud-dataproc/google/cloud/dataproc_v1/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
ShieldedInstanceConfig,
SoftwareConfig,
StartClusterRequest,
StartupConfig,
StopClusterRequest,
UpdateClusterRequest,
VirtualClusterConfig,
Expand Down Expand Up @@ -305,6 +306,7 @@
"SparkSqlBatch",
"SparkSqlJob",
"StartClusterRequest",
"StartupConfig",
"StopClusterRequest",
"SubmitJobRequest",
"TemplateParameter",
Expand Down
2 changes: 2 additions & 0 deletions packages/google-cloud-dataproc/google/cloud/dataproc_v1/types/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
ShieldedInstanceConfig,
SoftwareConfig,
StartClusterRequest,
StartupConfig,
StopClusterRequest,
UpdateClusterRequest,
VirtualClusterConfig,
Expand Down Expand Up @@ -217,6 +218,7 @@
"ShieldedInstanceConfig",
"SoftwareConfig",
"StartClusterRequest",
"StartupConfig",
"StopClusterRequest",
"UpdateClusterRequest",
"VirtualClusterConfig",
Expand Down
38 changes: 38 additions & 0 deletions packages/google-cloud-dataproc/google/cloud/dataproc_v1/types/clusters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
"ShieldedInstanceConfig",
"ConfidentialInstanceConfig",
"InstanceGroupConfig",
"StartupConfig",
"InstanceReference",
"ManagedGroupConfig",
"InstanceFlexibilityPolicy",
Expand Down Expand Up @@ -856,6 +857,10 @@ class InstanceGroupConfig(proto.Message):
Optional. Instance flexibility Policy
allowing a mixture of VM shapes and provisioning
models.
startup_config (google.cloud.dataproc_v1.types.StartupConfig):
Optional. Configuration to handle the startup
of instances during cluster create and update
process.
"""

class Preemptibility(proto.Enum):
Expand Down Expand Up @@ -952,6 +957,39 @@ class Preemptibility(proto.Enum):
number=13,
message="InstanceFlexibilityPolicy",
)
startup_config: "StartupConfig" = proto.Field(
proto.MESSAGE,
number=14,
message="StartupConfig",
)


class StartupConfig(proto.Message):
r"""Configuration to handle the startup of instances during
cluster create and update process.


.. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields

Attributes:
required_registration_fraction (float):
Optional. The config setting to enable cluster creation/
updation to be successful only after
required_registration_fraction of instances are up and
running. This configuration is applicable to only secondary
workers for now. The cluster will fail if
required_registration_fraction of instances are not
available. This will include instance creation, agent
registration, and service registration (if enabled).

This field is a member of `oneof`_ ``_required_registration_fraction``.
"""

required_registration_fraction: float = proto.Field(
proto.DOUBLE,
number=1,
optional=True,
)


class InstanceReference(proto.Message):
Expand Down
3 changes: 3 additions & 0 deletions packages/google-cloud-dataproc/noxfile.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,10 @@

BLACK_VERSION = "black[jupyter]==23.7.0"
ISORT_VERSION = "isort==5.11.0"

LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"]


DEFAULT_PYTHON_VERSION = "3.9"

UNIT_TEST_PYTHON_VERSIONS: List[str] = ["3.7", "3.8", "3.9", "3.10", "3.11"]
Expand Down Expand Up @@ -89,6 +91,7 @@ def lint(session):
"--check",
*LINT_PATHS,
)

session.run("flake8", "google", "tests")


Expand Down
2 changes: 2 additions & 0 deletions packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_cluster_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2868,6 +2868,7 @@ def test_create_cluster_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"worker_config": {},
"secondary_worker_config": {},
Expand Down Expand Up @@ -3437,6 +3438,7 @@ def test_update_cluster_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"worker_config": {},
"secondary_worker_config": {},
Expand Down
1 change: 1 addition & 0 deletions packages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_node_group_controller.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1569,6 +1569,7 @@ def test_create_node_group_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"labels": {},
}
Expand Down
3 changes: 3 additions & 0 deletions ...ages/google-cloud-dataproc/tests/unit/gapic/dataproc_v1/test_workflow_template_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2834,6 +2834,7 @@ def test_create_workflow_template_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"worker_config": {},
"secondary_worker_config": {},
Expand Down Expand Up @@ -4026,6 +4027,7 @@ def test_instantiate_inline_workflow_template_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"worker_config": {},
"secondary_worker_config": {},
Expand Down Expand Up @@ -4658,6 +4660,7 @@ def test_update_workflow_template_rest(request_type):
{"machine_type": "machine_type_value", "vm_count": 875}
],
},
"startup_config": {"required_registration_fraction": 0.3216},
},
"worker_config": {},
"secondary_worker_config": {},
Expand Down