diff --git a/storage/storage_transfer/README.md b/storage/storage_transfer/README.md new file mode 100644 index 000000000000..2c3f7762ff3f --- /dev/null +++ b/storage/storage_transfer/README.md @@ -0,0 +1,51 @@ +# Transfer Service sample using Python + +This app creates two types of transfers using the Transfer Service tool. + +## Prerequisites + +1. Set up a project on Google Developers Console. + 1. Go to the [Developers Console](https://cloud.google.com/console) and create or select your project. + You will need the project ID later. +1. Within Developers Console, select APIs & auth > Credentials. + 1. Add a new JSON key credential for a service account. + 1. Set the environment variable GOOGLE_APPLICATION_CREDENTIALS to point to your JSON key. +1. Add the Storage Transfer service account, cloud-mobility@system.gserviceaccount.com as an + editor of your project. +1. Set up gcloud for application default credentials. + 1. `gcloud components update` + 1. `gcloud auth login` + 1. `gcloud config set project PROJECT_ID` +1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation). + +## Transfer from Amazon S3 to Google Cloud Storage + +Creating a one-time transfer from Amazon S3 to Google Cloud Storage. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. Set up data source. + 1. Go to AWS Management Console and create a bucket. + 1. Under Security Credentials, create an IAM User with access to the bucket. + 1. Create an Access Key for the user. Note the Access Key ID and Secret Access Key. +1. In aws_request.py, fill in the Transfer Job JSON template with relevant values. +1. Run with `python aws_request.py` + 1. Note the job ID in the returned Transfer Job. + +## Transfer data from a standard Cloud Storage bucket to a Cloud Storage Nearline bucket + +Creating a daily transfer from a standard Cloud Storage bucket to a Cloud Storage Nearline +bucket for files untouched for 30 days. +1. Set up data sink. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. + 1. Select Nearline for Storage Class. +1. Set up data source. + 1. Go to the Developers Console and create a bucket under Cloud Storage > Storage Browser. +1. In nearline_request.py, fill in the Transfer Job JSON template with relevant values. +1. Run with `python nearline_request.py` + 1. Note the job ID in the returned Transfer Job. + +## Checking the status of a transfer + +1. In transfer_check.py, fill in the Transfer Job JSON template with relevant values. + Use the Job Name you recorded earlier. +1. Run with `python transfer_check.py` diff --git a/storage/storage_transfer/aws_request.py b/storage/storage_transfer/aws_request.py new file mode 100644 index 000000000000..84714de47934 --- /dev/null +++ b/storage/storage_transfer/aws_request.py @@ -0,0 +1,68 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + + +def main(): + """Create a one-off transfer from Amazon S3 to GCS.""" + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + # Edit this template with desired parameters. + # Specify times below using US Pacific Time Zone. + transfer_job = ''' + { + "description": "YOUR DESCRIPTION", + "status": "ENABLED", + "projectId": "YOUR_PROJECT_ID", + "schedule": { + "scheduleStartDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "scheduleEndDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "startTimeOfDay": { + "hours": 0, + "minutes": 0 + } + }, + "transferSpec": { + "awsS3DataSource": { + "bucketName": "YOUR_SOURCE_BUCKET", + "awsAccessKey": { + "accessKeyId": "YOUR_ACCESS_KEY_ID", + "secretAccessKey": "YOUR_SECRET_ACCESS_KEY" + } + }, + "gcsDataSink": { + "bucketName": "YOUR_SINK_BUCKET" + } + } + } + ''' + + result = transfer_service_client.transferJobs().create(body=json.loads( + transfer_job)).execute() + logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) + +if __name__ == '__main__': + main() diff --git a/storage/storage_transfer/create_client.py b/storage/storage_transfer/create_client.py new file mode 100644 index 000000000000..c5708950918c --- /dev/null +++ b/storage/storage_transfer/create_client.py @@ -0,0 +1,27 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging + +from apiclient import discovery +from oauth2client.client import GoogleCredentials + +CLOUD_SCOPES = 'https://www.googleapis.com/auth/cloud-platform' + + +def create_transfer_client(): + """Create a transfer client.""" + + logging.getLogger().setLevel(logging.DEBUG) + credentials = GoogleCredentials.get_application_default() + return discovery.build('storagetransfer', 'v1', credentials=credentials) diff --git a/storage/storage_transfer/nearline_request.py b/storage/storage_transfer/nearline_request.py new file mode 100644 index 000000000000..2e16b52252e7 --- /dev/null +++ b/storage/storage_transfer/nearline_request.py @@ -0,0 +1,64 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + + +def main(): + """Transfer from standard Cloud Storage to Cloud Storage Nearline.""" + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + # Edit this template with desired parameters. + # Specify times below using US Pacific Time Zone. + transfer_job = ''' + { + "description": "YOUR DESCRIPTION", + "status": "ENABLED", + "projectId": "YOUR_PROJECT_ID", + "schedule": { + "scheduleStartDate": { + "day": 1, + "month": 1, + "year": 2015 + }, + "startTimeOfDay": { + "hours": 1, + "minutes": 1 + } + }, + "transferSpec": { + "gcsDataSource": { + "bucketName": "YOUR_SOURCE_BUCKET" + }, + "gcsDataSink": { + "bucketName": "YOUR_SINK_BUCKET" + }, + "objectConditions": { + "minTimeElapsedSinceLastModification": "2592000s" + }, + "transferOptions": { + "deleteObjectsFromSourceAfterTransfer": true + } + } + } + ''' + result = transfer_service_client.transferJobs().create(body=json.loads( + transfer_job)).execute() + logging.info('Returned transferJob: %s', json.dumps(result, indent=4)) + +if __name__ == '__main__': + main() diff --git a/storage/storage_transfer/test_aws_request.py b/storage/storage_transfer/test_aws_request.py new file mode 100644 index 000000000000..f37a7c512ab6 --- /dev/null +++ b/storage/storage_transfer/test_aws_request.py @@ -0,0 +1,39 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from aws_request import main + +from mock import Mock +from mock import patch + + +class AwsRequestTestCase(unittest.TestCase): + """A test case for creating a TransferJob from AWS S3.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.aws_request.create_client') + self.mock_create_client = patcher1.start() + self.addCleanup(patcher1.stop) + self.mock_client = Mock(spec=['transferJobs']) + self.mock_create_client.create_transfer_client.return_value = \ + self.mock_client + + def test_create_aws_request(self): + execute = self.mock_client.transferJobs.return_value.create.return_value \ + .execute + execute.return_value = "" + main() + execute.assert_called_with() diff --git a/storage/storage_transfer/test_create_client.py b/storage/storage_transfer/test_create_client.py new file mode 100644 index 000000000000..c74cde2e16eb --- /dev/null +++ b/storage/storage_transfer/test_create_client.py @@ -0,0 +1,44 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +import create_client + +from mock import Mock +from mock import patch +from oauth2client.client import GoogleCredentials + + +class CheckCreateClientTestCase(unittest.TestCase): + """A test case for client creation.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.create_client.GoogleCredentials') + patcher2 = patch( + 'storage.storage_transfer.create_client.discovery.build') + self.mock_google_credentials = patcher1.start() + self.mock_discovery = patcher2.start() + self.addCleanup(patcher1.stop) + self.addCleanup(patcher2.stop) + + self.mock_credentials = Mock(spec=GoogleCredentials) + self.mock_google_credentials.get_application_default.return_value = \ + self.mock_credentials + + def test_create_client(self): + create_client.create_transfer_client() + self.mock_discovery.assert_called_with( + 'storagetransfer', 'v1', + credentials=self.mock_credentials) diff --git a/storage/storage_transfer/test_nearline_request.py b/storage/storage_transfer/test_nearline_request.py new file mode 100644 index 000000000000..f70b45e5d5da --- /dev/null +++ b/storage/storage_transfer/test_nearline_request.py @@ -0,0 +1,39 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from mock import Mock +from mock import patch + +from nearline_request import main + + +class NearlineRequestTestCase(unittest.TestCase): + """A test case for creating a TransferJob to Nearline for old files.""" + + def setUp(self): + patcher1 = patch( + 'storage.storage_transfer.nearline_request.create_client') + self.mock_create_client = patcher1.start() + self.addCleanup(patcher1.stop) + self.mock_client = Mock(spec=['transferJobs']) + self.mock_create_client.create_transfer_client.return_value = \ + self.mock_client + + def test_create_nearline_request(self): + execute = self.mock_client.transferJobs.return_value.create.return_value \ + .execute + execute.return_value = "" + main() + execute.assert_called_with() diff --git a/storage/storage_transfer/test_transfer_check.py b/storage/storage_transfer/test_transfer_check.py new file mode 100644 index 000000000000..cd3c6886979c --- /dev/null +++ b/storage/storage_transfer/test_transfer_check.py @@ -0,0 +1,31 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import unittest + +from mock import Mock + +from transfer_check import check_operation + + +class CheckTransferTestCase(unittest.TestCase): + """A test case for querying transfer job completion.""" + + def test_check_operation(self): + mock_client = Mock(spec=['transferOperations']) + execute = mock_client.transferOperations.return_value.list.return_value \ + .execute + project_id = "" + job_name = "" + check_operation(mock_client, project_id, job_name) + execute.assert_called_with() diff --git a/storage/storage_transfer/transfer_check.py b/storage/storage_transfer/transfer_check.py new file mode 100644 index 000000000000..5b1029819bb9 --- /dev/null +++ b/storage/storage_transfer/transfer_check.py @@ -0,0 +1,44 @@ +# Copyright 2015, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import json +import logging + +import create_client + +# Edit these values with desired parameters. +PROJECT_ID = 'YOUR_PROJECT_ID' +JOB_NAME = 'YOUR_JOB_NAME' + + +def check_operation(transfer_service_client, project_id, job_name): + """Review the transfer operations associated with a transfer job.""" + filterString = ( + '{{"project_id": "{project_id}", ' + '"job_names": ["{job_name}"]}}').format( + project_id=project_id, job_name=job_name) + return transfer_service_client.transferOperations().list( + name="transferOperations", + filter=filterString).execute() + + +def main(): + logging.getLogger().setLevel(logging.DEBUG) + transfer_service_client = create_client.create_transfer_client() + + result = check_operation(transfer_service_client, PROJECT_ID, JOB_NAME) + logging.info('Result of transferOperations/list: %s', + json.dumps(result, indent=4, sort_keys=True)) + +if __name__ == '__main__': + main()