From 8ca8a43b3b8369c905af3dbd16c38c062907db1d Mon Sep 17 00:00:00 2001 From: bradmiro Date: Tue, 12 Nov 2019 14:48:03 -0500 Subject: [PATCH 1/3] adding sample for cluster create --- dataproc/create_cluster.py | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 dataproc/create_cluster.py diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py new file mode 100644 index 00000000000..d756005d9da --- /dev/null +++ b/dataproc/create_cluster.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python + +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +def create_cluster(project_id, region, cluster_name): + # [START create_cluster] + from google.cloud import dataproc_v1 + + # TODO(developer): Uncomment and set the following variables + # project_id = 'YOUR_PROJECT_ID' + # region = 'YOUR_CLUSTER_REGION' + # cluster_name = 'YOUR_CLUSTER_NAME' + + # Create client + client = dataproc_v1.ClusterControllerClient({ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) + }) + + # Create cluster config + cluster = { + 'project_id': project, + 'cluster_name': cluster_name, + 'config': { + 'master_config': { + 'num_instances': 1, + 'machine_type_uri': 'n1-standard-1' + }, + 'worker_config': { + 'num_instances': 2, + 'machine_type_uri': 'n1-standard-1' + } + } + } + + # Define a callback + def callback(operation_future): + result = operation_future.result() + print(result) + + # Submit cluster creation request + response = client.create_cluster(project_id, region, cluster) + + # Add callback to cluster creation request + response.add_done_callback(callback) + + print(response.metadata()) + # [END create_cluster] + + From d8e174f3e5c0b67cc946d87ad3ad7565934e47f3 Mon Sep 17 00:00:00 2001 From: bradmiro Date: Tue, 12 Nov 2019 14:49:14 -0500 Subject: [PATCH 2/3] small fix --- dataproc/create_cluster.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index d756005d9da..96e4bde2e49 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - def create_cluster(project_id, region, cluster_name): # [START create_cluster] from google.cloud import dataproc_v1 From 6721673c291423fc0af229ad993d160de593764a Mon Sep 17 00:00:00 2001 From: bradmiro Date: Fri, 15 Nov 2019 16:58:25 -0500 Subject: [PATCH 3/3] Add create cluster samples --- dataproc/create_cluster.py | 29 +++++++++------------- dataproc/create_cluster_test.py | 44 +++++++++++++++++++++++++++++++++ dataproc/requirements.txt | 2 +- 3 files changed, 56 insertions(+), 19 deletions(-) create mode 100644 dataproc/create_cluster_test.py diff --git a/dataproc/create_cluster.py b/dataproc/create_cluster.py index 96e4bde2e49..112656320ed 100644 --- a/dataproc/create_cluster.py +++ b/dataproc/create_cluster.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def create_cluster(project_id, region, cluster_name): # [START create_cluster] from google.cloud import dataproc_v1 @@ -23,14 +24,14 @@ def create_cluster(project_id, region, cluster_name): # region = 'YOUR_CLUSTER_REGION' # cluster_name = 'YOUR_CLUSTER_NAME' - # Create client - client = dataproc_v1.ClusterControllerClient({ + # Create a client with the endpoint set to the desired cluster region + client = dataproc_v1.ClusterControllerClient(client_options={ 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(region) }) - # Create cluster config + # Create the cluster config cluster = { - 'project_id': project, + 'project_id': project_id, 'cluster_name': cluster_name, 'config': { 'master_config': { @@ -44,18 +45,10 @@ def create_cluster(project_id, region, cluster_name): } } - # Define a callback - def callback(operation_future): - result = operation_future.result() - print(result) - - # Submit cluster creation request - response = client.create_cluster(project_id, region, cluster) - - # Add callback to cluster creation request - response.add_done_callback(callback) - - print(response.metadata()) - # [END create_cluster] - + # Create the cluster + operation = client.create_cluster(project_id, region, cluster) + result = operation.result() + # Output a success message + print('Cluster created successfully: {}'.format(result.cluster_name)) + # [END create_cluster] diff --git a/dataproc/create_cluster_test.py b/dataproc/create_cluster_test.py new file mode 100644 index 00000000000..7e8f882df65 --- /dev/null +++ b/dataproc/create_cluster_test.py @@ -0,0 +1,44 @@ +# Copyright 2019 Google, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import uuid +import pytest + +from google.cloud import dataproc + +import create_cluster + +PROJECT_ID = os.environ['GCLOUD_PROJECT'] +REGION = 'us-central1' +CLUSTER_NAME = 'test-cluster-{}'.format(str(uuid.uuid4())) + + +@pytest.fixture(autouse=True) +def teardown(): + yield + + client = dataproc.ClusterControllerClient(client_options={ + 'api_endpoint': '{}-dataproc.googleapis.com:443'.format(REGION) + }) + # Client library function + client.delete_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + +def test_cluster_create(capsys): + # Wrapper function for client library function + create_cluster.create_cluster(PROJECT_ID, REGION, CLUSTER_NAME) + + out, _ = capsys.readouterr() + assert CLUSTER_NAME in out diff --git a/dataproc/requirements.txt b/dataproc/requirements.txt index 81a0a72bf2b..0ffe752a6ea 100644 --- a/dataproc/requirements.txt +++ b/dataproc/requirements.txt @@ -3,4 +3,4 @@ google-auth==1.6.3 google-auth-httplib2==0.0.3 google-cloud==0.34.0 google-cloud-storage==1.19.1 -google-cloud-dataproc==0.5.0 +google-cloud-dataproc==0.6.1