From 85f1201aa72b05ecf584b0fd549e1fd756fc2f59 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Fri, 24 Aug 2018 09:18:02 -0700 Subject: [PATCH 1/3] Update hardcoded values in StorageIT tests to be usable on any project. --- .../google-cloud-storage/pom.xml | 20 ++ .../cloud/storage/it/ITStorageTest.java | 181 +++++++++++++++--- 2 files changed, 171 insertions(+), 30 deletions(-) diff --git a/google-cloud-clients/google-cloud-storage/pom.xml b/google-cloud-clients/google-cloud-storage/pom.xml index 90004104a330..1c4c146da489 100644 --- a/google-cloud-clients/google-cloud-storage/pom.xml +++ b/google-cloud-clients/google-cloud-storage/pom.xml @@ -41,6 +41,26 @@ + + io.grpc + grpc-netty-shaded + test + + + io.grpc + grpc-auth + test + + + com.google.api.grpc + grpc-google-cloud-kms-v1 + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + ${project.groupId} google-cloud-core diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index ce873f9ea717..a12b23d26b7a 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -27,6 +27,16 @@ import static org.junit.Assert.fail; import com.google.api.gax.paging.Page; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.kms.v1.CreateCryptoKeyRequest; +import com.google.cloud.kms.v1.CreateKeyRingRequest; +import com.google.cloud.kms.v1.CryptoKeyName; +import com.google.cloud.kms.v1.CryptoKey; +import com.google.cloud.kms.v1.GetCryptoKeyRequest; +import com.google.cloud.kms.v1.GetKeyRingRequest; +import com.google.cloud.kms.v1.KeyManagementServiceGrpc.KeyManagementServiceBlockingStub; +import com.google.cloud.kms.v1.KeyManagementServiceGrpc; +import com.google.cloud.kms.v1.KeyRingName; import com.google.cloud.Identity; import com.google.cloud.Policy; import com.google.cloud.ReadChannel; @@ -60,6 +70,7 @@ import com.google.common.collect.Lists; import com.google.common.io.BaseEncoding; import com.google.common.io.ByteStreams; +import com.google.iam.v1.Binding; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -68,8 +79,6 @@ import java.net.URLConnection; import java.nio.ByteBuffer; import java.security.Key; -import java.security.NoSuchAlgorithmException; -import java.security.spec.InvalidKeySpecException; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -85,6 +94,17 @@ import java.util.logging.Logger; import java.util.zip.GZIPInputStream; import javax.crypto.spec.SecretKeySpec; + +import com.google.iam.v1.IAMPolicyGrpc; +import com.google.iam.v1.SetIamPolicyRequest; +import io.grpc.auth.MoreCallCredentials; +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.Metadata; +import io.grpc.Status; +import io.grpc.StatusRuntimeException; +import io.grpc.stub.MetadataUtils; + import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -94,6 +114,9 @@ public class ITStorageTest { private static RemoteStorageHelper remoteStorageHelper; private static Storage storage; + private static String kmsKeyOneResourcePath; + private static String kmsKeyTwoResourcePath; + private static final Logger log = Logger.getLogger(ITStorageTest.class.getName()); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static final String CONTENT_TYPE = "text/plain"; @@ -107,18 +130,23 @@ public class ITStorageTest { private static final byte[] COMPRESSED_CONTENT = BaseEncoding.base64() .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); - private static final String SERVICE_ACCOUNT_EMAIL = "gcloud-devel@gs-project-accounts.iam.gserviceaccount.com"; - private static final String KMS_KEY_NAME_1 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key"; - private static final String KMS_KEY_NAME_2 = "projects/gcloud-devel/locations/us/keyRings/gcs_kms_key_ring_us/cryptoKeys/key2"; + private static final String SERVICE_ACCOUNT_EMAIL_SUFFIX = "@gs-project-accounts.iam.gserviceaccount.com"; + private static final String KMS_KEY_RING_NAME = "gcs_kms_key_ring"; + private static final String KMS_KEY_RING_LOCATION = "us"; + private static final String KMS_KEY_ONE_NAME = "gcs_kms_key_one"; + private static final String KMS_KEY_TWO_NAME = "gcs_kms_key_two"; @BeforeClass - public static void beforeClass() throws NoSuchAlgorithmException, InvalidKeySpecException { + public static void beforeClass() throws IOException { remoteStorageHelper = RemoteStorageHelper.create(); storage = remoteStorageHelper.getOptions().getService(); storage.create( BucketInfo.newBuilder(BUCKET) .setDeleteRules(Collections.singleton(new BucketInfo.AgeDeleteRule(1))) .build()); + + // Prepare KMS KeyRing for CMEK tests + prepareKmsKeys(); } @AfterClass @@ -137,6 +165,99 @@ public static void afterClass() throws ExecutionException, InterruptedException } } + private static void prepareKmsKeys() throws IOException { + String projectId = remoteStorageHelper.getOptions().getProjectId(); + GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); + ManagedChannel kmsChannel = ManagedChannelBuilder.forTarget("cloudkms.googleapis.com:443") + .build(); + KeyManagementServiceBlockingStub keyManagementServiceBlockingStub = KeyManagementServiceGrpc + .newBlockingStub(kmsChannel).withCallCredentials(MoreCallCredentials.from(credentials)); + Metadata requestParamsHeader = new Metadata(); + Metadata.Key requestParamsKey = + Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); + + String kmsKeyRingResourcePath = KeyRingName + .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME).toString(); + try { + // Attempt to Get KeyRing + GetKeyRingRequest getKeyRingRequest = GetKeyRingRequest.newBuilder().setName(kmsKeyRingResourcePath) + .build(); + requestParamsHeader.put(requestParamsKey, "name="+kmsKeyRingResourcePath); + KeyManagementServiceBlockingStub serviceBlockingStubForGetKeyRing = MetadataUtils + .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); + serviceBlockingStubForGetKeyRing.getKeyRing(getKeyRingRequest); + } catch (StatusRuntimeException ex) { + if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { + // Create KmsKeyRing + String keyRingParent = "parent=projects/" + projectId + "/locations/" + KMS_KEY_RING_LOCATION; + CreateKeyRingRequest createKeyRingRequest = CreateKeyRingRequest.newBuilder() + .setParent(keyRingParent) + .setKeyRingId(KMS_KEY_RING_NAME).build(); + requestParamsHeader.put(requestParamsKey, keyRingParent); + KeyManagementServiceBlockingStub serviceBlockingStubForCreateKeyRing = MetadataUtils + .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); + serviceBlockingStubForCreateKeyRing.createKeyRing(createKeyRingRequest); + } else { + throw ex; + } + } + + ServiceAccount serviceAccount = storage.getServiceAccount(projectId); + IAMPolicyGrpc.IAMPolicyBlockingStub iamPolicyBlockingStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + Binding binding = Binding.newBuilder().setRole("roles/cloudkms.cryptoKeyEncrypterDecrypter") + .addMembers("serviceAccount:"+serviceAccount.getEmail()).build(); + com.google.iam.v1.Policy policy = com.google.iam.v1.Policy.newBuilder().addBindings(binding).build(); + SetIamPolicyRequest setIamPolicyRequest = SetIamPolicyRequest.newBuilder().setResource(kmsKeyRingResourcePath) + .setPolicy(policy).build(); + requestParamsHeader.put(requestParamsKey, "parent="+kmsKeyRingResourcePath); + iamPolicyBlockingStub = MetadataUtils.attachHeaders(iamPolicyBlockingStub, requestParamsHeader); + iamPolicyBlockingStub.setIamPolicy(setIamPolicyRequest); + + kmsKeyOneResourcePath = CryptoKeyName + .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_ONE_NAME).toString(); + try { + // Attempt to Get CryptoKey + GetCryptoKeyRequest getCryptoKeyOneRequest = GetCryptoKeyRequest.newBuilder() + .setName(kmsKeyOneResourcePath).build(); + keyManagementServiceBlockingStub.getCryptoKey(getCryptoKeyOneRequest); + } catch(StatusRuntimeException ex) { + if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { + CryptoKey cryptoKey = CryptoKey.newBuilder().setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT).build(); + CreateCryptoKeyRequest createCryptoKeyOneRequest = CreateCryptoKeyRequest.newBuilder() + .setCryptoKeyId(KMS_KEY_ONE_NAME).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); + KeyManagementServiceBlockingStub serviceBlockingStubForCreateCryptoKey = MetadataUtils + .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); + serviceBlockingStubForCreateCryptoKey.createCryptoKey(createCryptoKeyOneRequest); + } else { + throw ex; + } + } + + kmsKeyTwoResourcePath = CryptoKeyName + .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_TWO_NAME).toString(); + try { + // Attempt to Get CryptoKey + GetCryptoKeyRequest getCryptoKeyTwoRequest = GetCryptoKeyRequest.newBuilder() + .setName(kmsKeyTwoResourcePath).build(); + keyManagementServiceBlockingStub.getCryptoKey(getCryptoKeyTwoRequest); + } catch(StatusRuntimeException ex) { + if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { + // Create CryptoKeyTwo + CryptoKey cryptoKey = CryptoKey.newBuilder().setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT).build(); + CreateCryptoKeyRequest createCryptoKeyTwoRequest = CreateCryptoKeyRequest.newBuilder() + .setCryptoKeyId(KMS_KEY_TWO_NAME).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); + KeyManagementServiceGrpc.KeyManagementServiceBlockingStub serviceBlockingStubForCreateCryptoKey = MetadataUtils + .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); + serviceBlockingStubForCreateCryptoKey.createCryptoKey(createCryptoKeyTwoRequest); + } else { + throw ex; + } + } + } + @Test(timeout = 5000) public void testListBuckets() throws InterruptedException { Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(BUCKET), @@ -183,10 +304,10 @@ public void testGetBucketEmptyFields() { public void testClearBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); try { - assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); + assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(null).build().update(); assertNull(updatedBucket.getDefaultKmsKeyName()); } finally { @@ -198,12 +319,12 @@ public void testClearBucketDefaultKmsKeyName() throws ExecutionException, Interr public void testUpdateBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); try { - assertEquals(KMS_KEY_NAME_1, remoteBucket.getDefaultKmsKeyName()); - Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(KMS_KEY_NAME_2).build().update(); - assertEquals(KMS_KEY_NAME_2, updatedBucket.getDefaultKmsKeyName()); + assertEquals(kmsKeyOneResourcePath, remoteBucket.getDefaultKmsKeyName()); + Bucket updatedBucket = remoteBucket.toBuilder().setDefaultKmsKeyName(kmsKeyTwoResourcePath).build().update(); + assertEquals(kmsKeyTwoResourcePath, updatedBucket.getDefaultKmsKeyName()); } finally { RemoteStorageHelper.forceDelete(storage, bucketName, 5, TimeUnit.SECONDS); } @@ -240,12 +361,12 @@ public void testCreateBlobWithEncryptionKey() { public void testCreateBlobWithKmsKeyName() { String blobName = "test-create-with-kms-key-name-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); - Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Blob remoteBlob = storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); assertNotNull(remoteBlob); assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); byte[] readBytes = storage.readAllBytes(BUCKET, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } @@ -256,7 +377,7 @@ public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { String blobName = "test-create-with-kms-key-name-blob"; BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName).build(); storage.create(blob, BLOB_BYTE_CONTENT, Storage.BlobTargetOption.encryptionKey(KEY), - Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); fail("StorageException was expected"); // can't supply both. } catch (StorageException ex) { // expected @@ -267,8 +388,8 @@ public void testCreateBlobWithKmsKeyNameAndCustomerSuppliedKey() { public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket bucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); - assertEquals(bucket.getDefaultKmsKeyName(), KMS_KEY_NAME_1); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); + assertEquals(bucket.getDefaultKmsKeyName(), kmsKeyOneResourcePath); try { String blobName = "test-create-with-default-kms-key-name-blob"; @@ -278,7 +399,7 @@ public void testCreateBlobWithDefaultKmsKeyName() throws ExecutionException, Int assertEquals(blob.getBucket(), remoteBlob.getBucket()); assertEquals(blob.getName(), remoteBlob.getName()); assertNotNull(remoteBlob.getKmsKeyName()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); byte[] readBytes = storage.readAllBytes(bucketName, blobName); assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); } finally { @@ -375,11 +496,11 @@ public void testGetBlobKmsKeyNameField() { BlobInfo blob = BlobInfo.newBuilder(BUCKET, blobName) .setContentType(CONTENT_TYPE) .build(); - assertNotNull(storage.create(blob, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1))); + assertNotNull(storage.create(blob, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath))); Blob remoteBlob = storage.get(blob.getBlobId(), Storage.BlobGetOption.fields( BlobField.KMS_KEY_NAME)); assertEquals(blob.getBlobId(), remoteBlob.getBlobId()); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(remoteBlob.getContentType()); } @@ -479,8 +600,8 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { BlobInfo blob2 = BlobInfo.newBuilder(BUCKET, blobNames[1]) .setContentType(CONTENT_TYPE) .build(); - Blob remoteBlob1 = storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); - Blob remoteBlob2 = storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)); + Blob remoteBlob1 = storage.create(blob1, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); + Blob remoteBlob2 = storage.create(blob2, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)); assertNotNull(remoteBlob1); assertNotNull(remoteBlob2); Page page = storage.list(BUCKET, @@ -500,7 +621,7 @@ public void testListBlobsKmsKeySelectedFields() throws InterruptedException { Blob remoteBlob = iterator.next(); assertEquals(BUCKET, remoteBlob.getBucket()); assertTrue(blobSet.contains(remoteBlob.getName())); - assertTrue(remoteBlob.getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBlob.getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(remoteBlob.getContentType()); } } @@ -982,14 +1103,14 @@ public void testRotateFromCustomerEncryptionToKmsKey() { Storage.CopyRequest req = Storage.CopyRequest.newBuilder() .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) - .setTarget(target, Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) + .setTarget(target, Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)) .build(); CopyWriter copyWriter = storage.copy(req); assertEquals(BUCKET, copyWriter.getResult().getBucket()); assertEquals(targetBlobName, copyWriter.getResult().getName()); assertEquals(CONTENT_TYPE, copyWriter.getResult().getContentType()); assertNotNull(copyWriter.getResult().getKmsKeyName()); - assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(copyWriter.getResult().getKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertArrayEquals(BLOB_BYTE_CONTENT, copyWriter.getResult().getContent()); assertEquals(metadata, copyWriter.getResult().getMetadata()); assertTrue(copyWriter.isDone()); @@ -1014,7 +1135,7 @@ public void testRotateFromCustomerEncryptionToKmsKeyWithCustomerEncrytion() { .setSource(source) .setSourceOptions(Storage.BlobSourceOption.decryptionKey(BASE64_KEY)) .setTarget(target, Storage.BlobTargetOption.encryptionKey(KEY), - Storage.BlobTargetOption.kmsKeyName(KMS_KEY_NAME_1)) + Storage.BlobTargetOption.kmsKeyName(kmsKeyOneResourcePath)) .build(); storage.copy(req); fail("StorageException was expected"); @@ -1891,9 +2012,9 @@ public void testListBucketRequesterPaysFails() throws InterruptedException { public void testListBucketDefaultKmsKeyName() throws ExecutionException, InterruptedException { String bucketName = RemoteStorageHelper.generateBucketName(); Bucket remoteBucket = storage.create(BucketInfo.newBuilder(bucketName) - .setDefaultKmsKeyName(KMS_KEY_NAME_1).setLocation("US").build()); + .setDefaultKmsKeyName(kmsKeyOneResourcePath).setLocation(KMS_KEY_RING_LOCATION).build()); assertNotNull(remoteBucket); - assertTrue(remoteBucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(remoteBucket.getDefaultKmsKeyName().startsWith(kmsKeyOneResourcePath)); try { Iterator bucketIterator = storage.list(Storage.BucketListOption.prefix(bucketName), Storage.BucketListOption.fields(BucketField.ENCRYPTION)).iterateAll().iterator(); @@ -1906,7 +2027,7 @@ public void testListBucketDefaultKmsKeyName() throws ExecutionException, Interru Bucket bucket = bucketIterator.next(); assertTrue(bucket.getName().startsWith(bucketName)); assertNotNull(bucket.getDefaultKmsKeyName()); - assertTrue(bucket.getDefaultKmsKeyName().startsWith(KMS_KEY_NAME_1)); + assertTrue(bucket.getDefaultKmsKeyName().startsWith(kmsKeyOneResourcePath)); assertNull(bucket.getCreateTime()); assertNull(bucket.getSelfLink()); } @@ -1920,6 +2041,6 @@ public void testGetServiceAccount() throws InterruptedException { String projectId = remoteStorageHelper.getOptions().getProjectId(); ServiceAccount serviceAccount = storage.getServiceAccount(projectId); assertNotNull(serviceAccount); - assertEquals(SERVICE_ACCOUNT_EMAIL, serviceAccount.getEmail()); + assertTrue(serviceAccount.getEmail().endsWith(SERVICE_ACCOUNT_EMAIL_SUFFIX)); } } From a3859f385c7e7da5c6ebb98837826ed2fc8b4a5e Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 28 Aug 2018 21:18:59 -0700 Subject: [PATCH 2/3] address review comments --- .../cloud/storage/it/ITStorageTest.java | 108 +++++++++--------- 1 file changed, 53 insertions(+), 55 deletions(-) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index a12b23d26b7a..e7f369b4aca4 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -42,6 +42,7 @@ import com.google.cloud.ReadChannel; import com.google.cloud.RestorableState; import com.google.cloud.WriteChannel; +import com.google.cloud.kms.v1.LocationName; import com.google.cloud.storage.Acl; import com.google.cloud.storage.Acl.Role; import com.google.cloud.storage.Acl.User; @@ -113,7 +114,6 @@ public class ITStorageTest { private static RemoteStorageHelper remoteStorageHelper; private static Storage storage; - private static String kmsKeyOneResourcePath; private static String kmsKeyTwoResourcePath; @@ -131,7 +131,7 @@ public class ITStorageTest { .decode("H4sIAAAAAAAAAPNIzcnJV3DPz0/PSVVwzskvTVEILskvSkxPVQQA/LySchsAAAA="); private static final Map BUCKET_LABELS = ImmutableMap.of("label1", "value1"); private static final String SERVICE_ACCOUNT_EMAIL_SUFFIX = "@gs-project-accounts.iam.gserviceaccount.com"; - private static final String KMS_KEY_RING_NAME = "gcs_kms_key_ring"; + private static final String KMS_KEY_RING_NAME = "gcs_test_kms_key_ring"; private static final String KMS_KEY_RING_LOCATION = "us"; private static final String KMS_KEY_ONE_NAME = "gcs_kms_key_one"; private static final String KMS_KEY_TWO_NAME = "gcs_kms_key_two"; @@ -168,94 +168,92 @@ public static void afterClass() throws ExecutionException, InterruptedException private static void prepareKmsKeys() throws IOException { String projectId = remoteStorageHelper.getOptions().getProjectId(); GoogleCredentials credentials = GoogleCredentials.getApplicationDefault(); - ManagedChannel kmsChannel = ManagedChannelBuilder.forTarget("cloudkms.googleapis.com:443") - .build(); - KeyManagementServiceBlockingStub keyManagementServiceBlockingStub = KeyManagementServiceGrpc - .newBlockingStub(kmsChannel).withCallCredentials(MoreCallCredentials.from(credentials)); + ManagedChannel kmsChannel = ManagedChannelBuilder.forTarget("cloudkms.googleapis.com:443").build(); + KeyManagementServiceBlockingStub kmsStub = KeyManagementServiceGrpc.newBlockingStub(kmsChannel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + IAMPolicyGrpc.IAMPolicyBlockingStub iamStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) + .withCallCredentials(MoreCallCredentials.from(credentials)); + ensureKmsKeyRingExistsForTests(kmsStub, iamStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); + kmsKeyOneResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, + KMS_KEY_ONE_NAME); + kmsKeyTwoResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, + KMS_KEY_TWO_NAME); + } + + private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockingStub kmsStub, + IAMPolicyGrpc.IAMPolicyBlockingStub iamStub, String projectId, + String location, + String keyRingName) throws StatusRuntimeException { Metadata requestParamsHeader = new Metadata(); Metadata.Key requestParamsKey = Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); - - String kmsKeyRingResourcePath = KeyRingName - .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME).toString(); + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); try { // Attempt to Get KeyRing GetKeyRingRequest getKeyRingRequest = GetKeyRingRequest.newBuilder().setName(kmsKeyRingResourcePath) .build(); requestParamsHeader.put(requestParamsKey, "name="+kmsKeyRingResourcePath); - KeyManagementServiceBlockingStub serviceBlockingStubForGetKeyRing = MetadataUtils - .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); - serviceBlockingStubForGetKeyRing.getKeyRing(getKeyRingRequest); + KeyManagementServiceBlockingStub stubForGetKeyRing = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForGetKeyRing.getKeyRing(getKeyRingRequest); } catch (StatusRuntimeException ex) { if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { // Create KmsKeyRing - String keyRingParent = "parent=projects/" + projectId + "/locations/" + KMS_KEY_RING_LOCATION; + String keyRingParent = LocationName.of(projectId, location).toString(); CreateKeyRingRequest createKeyRingRequest = CreateKeyRingRequest.newBuilder() .setParent(keyRingParent) - .setKeyRingId(KMS_KEY_RING_NAME).build(); - requestParamsHeader.put(requestParamsKey, keyRingParent); - KeyManagementServiceBlockingStub serviceBlockingStubForCreateKeyRing = MetadataUtils - .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); - serviceBlockingStubForCreateKeyRing.createKeyRing(createKeyRingRequest); + .setKeyRingId(keyRingName).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + keyRingParent); + KeyManagementServiceBlockingStub stubForCreateKeyRing = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForCreateKeyRing.createKeyRing(createKeyRingRequest); } else { throw ex; } } - ServiceAccount serviceAccount = storage.getServiceAccount(projectId); - IAMPolicyGrpc.IAMPolicyBlockingStub iamPolicyBlockingStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) - .withCallCredentials(MoreCallCredentials.from(credentials)); Binding binding = Binding.newBuilder().setRole("roles/cloudkms.cryptoKeyEncrypterDecrypter") .addMembers("serviceAccount:"+serviceAccount.getEmail()).build(); com.google.iam.v1.Policy policy = com.google.iam.v1.Policy.newBuilder().addBindings(binding).build(); SetIamPolicyRequest setIamPolicyRequest = SetIamPolicyRequest.newBuilder().setResource(kmsKeyRingResourcePath) .setPolicy(policy).build(); - requestParamsHeader.put(requestParamsKey, "parent="+kmsKeyRingResourcePath); - iamPolicyBlockingStub = MetadataUtils.attachHeaders(iamPolicyBlockingStub, requestParamsHeader); - iamPolicyBlockingStub.setIamPolicy(setIamPolicyRequest); + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); + iamStub = MetadataUtils.attachHeaders(iamStub, requestParamsHeader); + iamStub.setIamPolicy(setIamPolicyRequest); - kmsKeyOneResourcePath = CryptoKeyName - .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_ONE_NAME).toString(); - try { - // Attempt to Get CryptoKey - GetCryptoKeyRequest getCryptoKeyOneRequest = GetCryptoKeyRequest.newBuilder() - .setName(kmsKeyOneResourcePath).build(); - keyManagementServiceBlockingStub.getCryptoKey(getCryptoKeyOneRequest); - } catch(StatusRuntimeException ex) { - if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { - CryptoKey cryptoKey = CryptoKey.newBuilder().setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT).build(); - CreateCryptoKeyRequest createCryptoKeyOneRequest = CreateCryptoKeyRequest.newBuilder() - .setCryptoKeyId(KMS_KEY_ONE_NAME).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); - requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); - KeyManagementServiceBlockingStub serviceBlockingStubForCreateCryptoKey = MetadataUtils - .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); - serviceBlockingStubForCreateCryptoKey.createCryptoKey(createCryptoKeyOneRequest); - } else { - throw ex; - } - } + return kmsKeyRingResourcePath; + } - kmsKeyTwoResourcePath = CryptoKeyName - .of(projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_TWO_NAME).toString(); + private static String ensureKmsKeyExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, String location, + String keyRingName, String keyName) throws StatusRuntimeException { + Metadata requestParamsHeader = new Metadata(); + Metadata.Key requestParamsKey = + Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); + String kmsKeyResourcePath = CryptoKeyName.of(projectId, location, keyRingName, keyName).toString(); try { // Attempt to Get CryptoKey - GetCryptoKeyRequest getCryptoKeyTwoRequest = GetCryptoKeyRequest.newBuilder() - .setName(kmsKeyTwoResourcePath).build(); - keyManagementServiceBlockingStub.getCryptoKey(getCryptoKeyTwoRequest); + requestParamsHeader.put(requestParamsKey, "name=" + kmsKeyResourcePath); + GetCryptoKeyRequest getCryptoKeyRequest = GetCryptoKeyRequest.newBuilder() + .setName(kmsKeyResourcePath).build(); + KeyManagementServiceGrpc.KeyManagementServiceBlockingStub stubForGetCryptoKey = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForGetCryptoKey.getCryptoKey(getCryptoKeyRequest); } catch(StatusRuntimeException ex) { if (ex.getStatus().getCode() == Status.Code.NOT_FOUND) { - // Create CryptoKeyTwo + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); CryptoKey cryptoKey = CryptoKey.newBuilder().setPurpose(CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT).build(); - CreateCryptoKeyRequest createCryptoKeyTwoRequest = CreateCryptoKeyRequest.newBuilder() - .setCryptoKeyId(KMS_KEY_TWO_NAME).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); + CreateCryptoKeyRequest createCryptoKeyRequest = CreateCryptoKeyRequest.newBuilder() + .setCryptoKeyId(keyName).setParent(kmsKeyRingResourcePath).setCryptoKey(cryptoKey).build(); + requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); - KeyManagementServiceGrpc.KeyManagementServiceBlockingStub serviceBlockingStubForCreateCryptoKey = MetadataUtils - .attachHeaders(keyManagementServiceBlockingStub, requestParamsHeader); - serviceBlockingStubForCreateCryptoKey.createCryptoKey(createCryptoKeyTwoRequest); + KeyManagementServiceGrpc.KeyManagementServiceBlockingStub stubForCreateCryptoKey = MetadataUtils + .attachHeaders(kmsStub, requestParamsHeader); + stubForCreateCryptoKey.createCryptoKey(createCryptoKeyRequest); } else { throw ex; } } + return kmsKeyResourcePath; } @Test(timeout = 5000) From ed6655779d1bd4f5dbe03e9f1acbfcc3c6488f38 Mon Sep 17 00:00:00 2001 From: Frank Natividad Date: Tue, 28 Aug 2018 21:26:19 -0700 Subject: [PATCH 3/3] address feedback comments --- .../cloud/storage/it/ITStorageTest.java | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java index e7f369b4aca4..a1330e946738 100644 --- a/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java +++ b/google-cloud-clients/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java @@ -116,7 +116,9 @@ public class ITStorageTest { private static Storage storage; private static String kmsKeyOneResourcePath; private static String kmsKeyTwoResourcePath; - + private static Metadata requestParamsHeader = new Metadata(); + private static Metadata.Key requestParamsKey = + Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); private static final Logger log = Logger.getLogger(ITStorageTest.class.getName()); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static final String CONTENT_TYPE = "text/plain"; @@ -173,20 +175,17 @@ private static void prepareKmsKeys() throws IOException { .withCallCredentials(MoreCallCredentials.from(credentials)); IAMPolicyGrpc.IAMPolicyBlockingStub iamStub = IAMPolicyGrpc.newBlockingStub(kmsChannel) .withCallCredentials(MoreCallCredentials.from(credentials)); - ensureKmsKeyRingExistsForTests(kmsStub, iamStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); + ensureKmsKeyRingExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); + ensureKmsKeyRingIamPermissionsForTests(iamStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME); kmsKeyOneResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_ONE_NAME); kmsKeyTwoResourcePath = ensureKmsKeyExistsForTests(kmsStub, projectId, KMS_KEY_RING_LOCATION, KMS_KEY_RING_NAME, KMS_KEY_TWO_NAME); } - private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockingStub kmsStub, - IAMPolicyGrpc.IAMPolicyBlockingStub iamStub, String projectId, + private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, String location, String keyRingName) throws StatusRuntimeException { - Metadata requestParamsHeader = new Metadata(); - Metadata.Key requestParamsKey = - Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); try { // Attempt to Get KeyRing @@ -211,7 +210,15 @@ private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockin throw ex; } } + + return kmsKeyRingResourcePath; + } + + private static void ensureKmsKeyRingIamPermissionsForTests(IAMPolicyGrpc.IAMPolicyBlockingStub iamStub, + String projectId, String location, + String keyRingName) throws StatusRuntimeException { ServiceAccount serviceAccount = storage.getServiceAccount(projectId); + String kmsKeyRingResourcePath = KeyRingName.of(projectId, location, keyRingName).toString(); Binding binding = Binding.newBuilder().setRole("roles/cloudkms.cryptoKeyEncrypterDecrypter") .addMembers("serviceAccount:"+serviceAccount.getEmail()).build(); com.google.iam.v1.Policy policy = com.google.iam.v1.Policy.newBuilder().addBindings(binding).build(); @@ -220,15 +227,11 @@ private static String ensureKmsKeyRingExistsForTests(KeyManagementServiceBlockin requestParamsHeader.put(requestParamsKey, "parent=" + kmsKeyRingResourcePath); iamStub = MetadataUtils.attachHeaders(iamStub, requestParamsHeader); iamStub.setIamPolicy(setIamPolicyRequest); - - return kmsKeyRingResourcePath; } - private static String ensureKmsKeyExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, String location, - String keyRingName, String keyName) throws StatusRuntimeException { - Metadata requestParamsHeader = new Metadata(); - Metadata.Key requestParamsKey = - Metadata.Key.of("x-goog-request-params", Metadata.ASCII_STRING_MARSHALLER); + private static String ensureKmsKeyExistsForTests(KeyManagementServiceBlockingStub kmsStub, String projectId, + String location, String keyRingName, + String keyName) throws StatusRuntimeException { String kmsKeyResourcePath = CryptoKeyName.of(projectId, location, keyRingName, keyName).toString(); try { // Attempt to Get CryptoKey