diff --git a/storage/api/customer_supplied_keys.py b/storage/api/customer_supplied_keys.py new file mode 100644 index 000000000000..5b5551e8daac --- /dev/null +++ b/storage/api/customer_supplied_keys.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python + +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Command-line sample app demonstrating customer-supplied encryption keys. + +This sample demonstrates uploading an object while supplying an encryption key, +retrieving that object's contents, and finally rotating that key to a new +value. + +This sample is used on this page: + + https://cloud.google.com/storage/docs/json_api/v1/json-api-python-samples + +For more information, see the README.md under /storage. +""" + +import argparse +import filecmp +import tempfile + +from googleapiclient import discovery +from googleapiclient import http +from oauth2client.client import GoogleCredentials + + +# You can (and should) generate your own encryption key. Here's a good way to +# accomplish this with Python: +# python -c \ +# 'import base64; import os; print(base64.encodestring(os.urandom(32)))' +# Although these keys are provided here for simplicity, please remember that it +# is a bad idea to store your encryption keys in your source code. +ENCRYPTION_KEY = '4RzDI0TeWa9M/nAvYH05qbCskPaSU/CFV5HeCxk0IUA=' + +# You can use openssl to quicly calculate the hash of any key. +# Try running this: +# openssl base64 -d <<< ENCRYPTION_KEY | openssl dgst -sha256 -binary \ +# | openssl base64 +KEY_HASH = 'aanjNC2nwso8e2FqcWILC3/Tt1YumvIwEj34kr6PRpI=' + +ANOTHER_ENCRYPTION_KEY = 'oevtavYZC+TfGtV86kJBKTeytXAm1s2r3xIqam+QPKM=' +ANOTHER_KEY_HASH = '/gd0N3k3MK0SEDxnUiaswl0FFv6+5PHpo+5KD5SBCeA=' + + +def create_service(): + """Creates the service object for calling the Cloud Storage API.""" + # Get the application default credentials. When running locally, these are + # available after running `gcloud init`. When running on compute + # engine, these are available from the environment. + credentials = GoogleCredentials.get_application_default() + + # Construct the service object for interacting with the Cloud Storage API - + # the 'storage' service, at version 'v1'. + # You can browse other available api services and versions here: + # https://developers.google.com/api-client-library/python/apis/ + return discovery.build('storage', 'v1', credentials=credentials) + + +def upload_object(bucket, filename, encryption_key, key_hash): + """Uploads an object, specifying a custom encryption key.""" + service = create_service() + + with open(filename, 'rb') as f: + request = service.objects().insert( + bucket=bucket, name=filename, + # You can also just set media_body=filename, but for the sake of + # demonstration, pass in the more generic file handle, which could + # very well be a StringIO or similar. + media_body=http.MediaIoBaseUpload(f, 'application/octet-stream')) + request.headers['x-goog-encryption-algorithm'] = 'AES256' + request.headers['x-goog-encryption-key'] = encryption_key + request.headers['x-goog-encryption-key-sha256'] = key_hash + + resp = request.execute() + + return resp + + +def download_object(bucket, obj, out_file, encryption_key, key_hash): + """Downloads an object protected by a custom encryption key.""" + service = create_service() + + request = service.objects().get_media(bucket=bucket, object=obj) + request.headers['x-goog-encryption-algorithm'] = 'AES256' + request.headers['x-goog-encryption-key'] = encryption_key + request.headers['x-goog-encryption-key-sha256'] = key_hash + + # Unfortunately, http.MediaIoBaseDownload overwrites HTTP headers, + # and so it cannot be used here. Instead, we shall download as a + # single request. + out_file.write(request.execute()) + + +def rotate_key(bucket, obj, current_encryption_key, current_key_hash, + new_encryption_key, new_key_hash): + """Changes the encryption key used to store an existing object.""" + service = create_service() + + request = service.objects().rewrite( + sourceBucket=bucket, sourceObject=obj, + destinationBucket=bucket, destinationObject=obj, + body={}) + + # For very large objects, calls to rewrite may not complete on the first + # call and may need to be resumed. + while True: + request.headers.update({ + 'x-goog-copy-source-encryption-algorithm': 'AES256', + 'x-goog-copy-source-encryption-key': current_encryption_key, + 'x-goog-copy-source-encryption-key-sha256': current_key_hash, + 'x-goog-encryption-algorithm': 'AES256', + 'x-goog-encryption-key': new_encryption_key, + 'x-goog-encryption-key-sha256': new_key_hash}) + + rewrite_response = request.execute() + + if rewrite_response['done']: + break + + print('Continuing rewrite call...') + request = service.objects().rewrite( + source_bucket=bucket, source_object=obj, + destination_bucket=bucket, destination_object=obj, + rewriteToken=rewrite_response['rewriteToken']) + rewrite_response.execute() + + +def main(bucket, filename): + print('Uploading object gs://{}/{}'.format(bucket, filename)) + upload_object(bucket, filename, ENCRYPTION_KEY, KEY_HASH) + print('Downloading it back') + with tempfile.NamedTemporaryFile(mode='w+b') as tmpfile: + download_object(bucket, filename, tmpfile, ENCRYPTION_KEY, KEY_HASH) + tmpfile.seek(0) + assert filecmp.cmp(filename, tmpfile.name), \ + 'Downloaded file has different content from the original file.' + print('Rotating its key') + rotate_key(bucket, filename, ENCRYPTION_KEY, KEY_HASH, + ANOTHER_ENCRYPTION_KEY, ANOTHER_KEY_HASH) + print('Done') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument('bucket', help='Your Cloud Storage bucket.') + parser.add_argument('filename', help='A file to upload and download.') + + args = parser.parse_args() + + main(args.bucket, args.filename) diff --git a/storage/api/customer_supplied_keys_test.py b/storage/api/customer_supplied_keys_test.py new file mode 100644 index 000000000000..ec4ec0b5031b --- /dev/null +++ b/storage/api/customer_supplied_keys_test.py @@ -0,0 +1,24 @@ +# Copyright 2016, Google, Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from customer_supplied_keys import main + + +def test_main(cloud_config, capsys): + main(cloud_config.storage_bucket, __file__) + out, err = capsys.readouterr() + + assert not re.search(r'Downloaded file [!]=', out) + assert re.search(r'Uploading.*Downloading.*Rotating.*Done', out, re.DOTALL)