From 0b35ffcc3aa98dcac5d81ec1221ddd36e9c17dce Mon Sep 17 00:00:00 2001 From: wei Date: Thu, 2 Nov 2017 16:02:58 -0700 Subject: [PATCH 1/3] Update AWS S3 client --- .gitignore | 1 + grails-app/conf/BuildConfig.groovy | 6 +- .../com/netflix/ice/basic/BasicManagers.java | 13 ++-- .../basic/BasicS3ApplicationGroupService.java | 4 +- src/java/com/netflix/ice/basic/MapDb.java | 6 +- src/java/com/netflix/ice/common/AwsUtils.java | 67 ++++++++++--------- .../ice/processor/BillingFileProcessor.java | 15 +++-- .../com/netflix/ice/reader/ReaderConfig.java | 2 +- 8 files changed, 62 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index a1d315f0..13810808 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ /*.tmproj /ivy* /eclipse +/.idea/* # default HSQL database files for production mode /prodDb.* diff --git a/grails-app/conf/BuildConfig.groovy b/grails-app/conf/BuildConfig.groovy index ac1d3fdd..2f33a62d 100644 --- a/grails-app/conf/BuildConfig.groovy +++ b/grails-app/conf/BuildConfig.groovy @@ -66,12 +66,12 @@ grails.project.dependency.resolution = { compile( // Amazon Web Services programmatic interface - 'com.amazonaws:aws-java-sdk:1.11.136', + 'com.amazonaws:aws-java-sdk:1.11.221', // Transitive dependencies of aws-java-sdk, but also used directly. // It would be great if we could upgrade httpcore and httpclient, but we can't until the AWS Java SDK // upgrades its dependencies. If we simply upgrade these, then some Amazon calls fail. - 'org.apache.httpcomponents:httpcore:4.4.4', - 'org.apache.httpcomponents:httpclient:4.5.2', + 'org.apache.httpcomponents:httpcore:4.4.7', + 'org.apache.httpcomponents:httpclient:4.5.3', // Explicitly including aws-java-sdk transitive dependencies 'org.codehaus.jackson:jackson-core-asl:1.8.9', diff --git a/src/java/com/netflix/ice/basic/BasicManagers.java b/src/java/com/netflix/ice/basic/BasicManagers.java index 33968a64..e3395dd4 100644 --- a/src/java/com/netflix/ice/basic/BasicManagers.java +++ b/src/java/com/netflix/ice/basic/BasicManagers.java @@ -17,13 +17,18 @@ */ package com.netflix.ice.basic; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.netflix.ice.common.*; +import com.netflix.ice.common.AwsUtils; +import com.netflix.ice.common.ConsolidateType; +import com.netflix.ice.common.Poller; import com.netflix.ice.processor.TagGroupWriter; -import com.netflix.ice.reader.*; +import com.netflix.ice.reader.DataManager; +import com.netflix.ice.reader.Managers; +import com.netflix.ice.reader.ReaderConfig; +import com.netflix.ice.reader.TagGroupManager; import com.netflix.ice.tag.Product; import com.netflix.ice.tag.Tag; @@ -91,7 +96,7 @@ private void doWork() { TreeMap usageManagers = Maps.newTreeMap(this.usageManagers); Set newProducts = Sets.newHashSet(); - AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); + AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).getObjectSummaries()) { String key = s3ObjectSummary.getKey(); Product product; diff --git a/src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java b/src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java index fe0ac833..6229e103 100644 --- a/src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java +++ b/src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java @@ -17,7 +17,7 @@ */ package com.netflix.ice.basic; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ObjectMetadata; import com.google.common.collect.Maps; import com.netflix.ice.common.AwsUtils; @@ -38,7 +38,7 @@ public class BasicS3ApplicationGroupService implements ApplicationGroupService { private final static Logger logger = LoggerFactory.getLogger(BasicS3ApplicationGroupService.class); - private AmazonS3Client s3Client; + private AmazonS3 s3Client; private ReaderConfig config; public void init() { diff --git a/src/java/com/netflix/ice/basic/MapDb.java b/src/java/com/netflix/ice/basic/MapDb.java index 08f1cf1a..ff9e545a 100644 --- a/src/java/com/netflix/ice/basic/MapDb.java +++ b/src/java/com/netflix/ice/basic/MapDb.java @@ -17,7 +17,7 @@ */ package com.netflix.ice.basic; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.netflix.ice.common.AwsUtils; import com.netflix.ice.processor.ProcessorConfig; @@ -47,7 +47,7 @@ public class MapDb { this.dbName = "db_" + name; File file = new File(config.localDir, dbName); if (!file.exists()) { - AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); + AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + this.dbName).getObjectSummaries()) { File dbFile = new File(config.localDir, s3ObjectSummary.getKey().substring(config.workS3BucketPrefix.length())); AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, dbFile); @@ -103,7 +103,7 @@ void commit() { } void upload() { - AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); + AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); File dir = new File(config.localDir); File[] files = dir.listFiles(new FilenameFilter() { diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index f179a520..9b7cb721 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -17,9 +17,12 @@ */ package com.netflix.ice.common; +import com.amazonaws.ClientConfiguration; import com.amazonaws.auth.AWSCredentialsProvider; -import com.amazonaws.auth.BasicSessionCredentials; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.*; import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; @@ -27,7 +30,6 @@ import com.amazonaws.services.securitytoken.model.Credentials; import com.amazonaws.services.simpledb.AmazonSimpleDBClient; import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; -import com.amazonaws.ClientConfiguration; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; @@ -57,7 +59,7 @@ public class AwsUtils { public static final DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HHa").withZone(DateTimeZone.UTC); public static long hourMillis = 3600000L; - private static AmazonS3Client s3Client; + private static AmazonS3 s3Client; private static AmazonSimpleEmailServiceClient emailServiceClient; private static AmazonSimpleDBClient simpleDBClient; private static AWSSecurityTokenServiceClient securityClient; @@ -89,23 +91,22 @@ public static void init(AWSCredentialsProvider credentialsProvider) { clientConfig = new ClientConfiguration(); String proxyHost = System.getProperty("https.proxyHost"); String proxyPort = System.getProperty("https.proxyPort"); - if(proxyHost != null && proxyPort != null) { + if (proxyHost != null && proxyPort != null) { clientConfig.setProxyHost(proxyHost); clientConfig.setProxyPort(Integer.parseInt(proxyPort)); } - s3Client = new AmazonS3Client(awsCredentialsProvider, clientConfig); - securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig); - if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) { - if ("global".equals(System.getProperty("EC2_REGION"))) { - s3Client.setEndpoint("s3.amazonaws.com"); - } - else { - s3Client.setEndpoint("s3-" + System.getProperty("EC2_REGION") + ".amazonaws.com"); - } + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder + .standard() + .withCredentials(credentialsProvider) + .withClientConfiguration(clientConfig); + if (System.getProperty("EC2_REGION") != null) { + amazonS3ClientBuilder.withRegion(System.getProperty("EC2_REGION")); } + s3Client = amazonS3ClientBuilder.build(); + securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig); } - public static AmazonS3Client getAmazonS3Client() { + public static AmazonS3 getAmazonS3Client() { return s3Client; } @@ -158,7 +159,7 @@ public static List listAllObjects(String bucket, String prefix) */ public static List listAllObjects(String bucket, String prefix, String accountId, String assumeRole, String externalId) { - AmazonS3Client s3Client = AwsUtils.s3Client; + AmazonS3 s3Client = AwsUtils.s3Client; try { ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix); @@ -166,11 +167,13 @@ public static List listAllObjects(String bucket, String prefix, if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) { Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId); - s3Client = new AmazonS3Client( - new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), - assumedCredentials.getSecretAccessKey(), - assumedCredentials.getSessionToken()), - clientConfig); + + BasicAWSCredentials awsCreds = new BasicAWSCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey()); + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder + .standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .withClientConfiguration(clientConfig); + s3Client = amazonS3ClientBuilder.build(); } ObjectListing page = null; @@ -260,20 +263,20 @@ public static long getLastModified(String bucketName, String fileKey) { public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file, long milles, String accountId, String assumeRole, String externalId) { - AmazonS3Client s3Client = AwsUtils.s3Client; + AmazonS3 s3Client = AwsUtils.s3Client; try { if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) { Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId); - s3Client = new AmazonS3Client( - new BasicSessionCredentials(assumedCredentials.getAccessKeyId(), - assumedCredentials.getSecretAccessKey(), - assumedCredentials.getSessionToken()), - clientConfig); - } - - if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) { - s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com"); + BasicAWSCredentials awsCreds = new BasicAWSCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey()); + AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder + .standard() + .withCredentials(new AWSStaticCredentialsProvider(awsCreds)) + .withClientConfiguration(clientConfig); + if (bucketFileRegion != null && !bucketFileRegion.isEmpty()) { + amazonS3ClientBuilder.setRegion(bucketFileRegion); + } + s3Client = amazonS3ClientBuilder.build(); } ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName()); @@ -333,7 +336,7 @@ private static boolean download(String bucketName, String fileKey, File file) { return download(s3Client, bucketName, fileKey, file); } - private static boolean download(AmazonS3Client s3Client, String bucketName, String fileKey, File file) { + private static boolean download(AmazonS3 s3Client, String bucketName, String fileKey, File file) { do { S3Object s3Object = s3Client.getObject(bucketName, fileKey); InputStream input = s3Object.getObjectContent(); diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java index 3d21790b..2593490b 100644 --- a/src/java/com/netflix/ice/processor/BillingFileProcessor.java +++ b/src/java/com/netflix/ice/processor/BillingFileProcessor.java @@ -17,7 +17,7 @@ */ package com.netflix.ice.processor; -import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; @@ -26,15 +26,16 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; -import com.netflix.ice.common.*; +import com.netflix.ice.common.AwsUtils; +import com.netflix.ice.common.Poller; +import com.netflix.ice.common.TagGroup; import com.netflix.ice.tag.Account; import com.netflix.ice.tag.Operation; import com.netflix.ice.tag.Product; -import com.netflix.ice.tag.Zone; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream; -import org.apache.commons.compress.archivers.ArchiveEntry; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.Months; @@ -648,12 +649,12 @@ private Map> getOndemandCosts } private void updateLastMillis(long millis, String filename) { - AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); + AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); s3Client.putObject(config.workS3BucketName, config.workS3BucketPrefix + filename, IOUtils.toInputStream(millis + ""), new ObjectMetadata()); } private Long getLastMillis(String filename) { - AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); + AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); InputStream in = null; try { in = s3Client.getObject(config.workS3BucketName, config.workS3BucketPrefix + filename).getObjectContent(); diff --git a/src/java/com/netflix/ice/reader/ReaderConfig.java b/src/java/com/netflix/ice/reader/ReaderConfig.java index aedb56aa..e3e52cd9 100644 --- a/src/java/com/netflix/ice/reader/ReaderConfig.java +++ b/src/java/com/netflix/ice/reader/ReaderConfig.java @@ -83,7 +83,7 @@ public ReaderConfig( ReaderConfig.instance = this; -// AmazonS3Client s3Client = AwsUtils.getAmazonS3Client(); +// AmazonS3 s3Client = AwsUtils.getAmazonS3Client(); // logger.info("Deleting all files..."); // List objectSummariesToDelete = AwsUtils.listAllObjects(instance.workS3BucketName, instance.workS3BucketPrefix); // for (S3ObjectSummary objectSummary : objectSummariesToDelete) { From 08301346c2124d9455e2f5f57ed1158d3f3f553b Mon Sep 17 00:00:00 2001 From: wei Date: Thu, 2 Nov 2017 16:05:48 -0700 Subject: [PATCH 2/3] Update AWS EC2 client --- .../ice/basic/BasicReservationService.java | 22 +++++++++----- .../processor/ReservationCapacityPoller.java | 30 +++++++++---------- 2 files changed, 30 insertions(+), 22 deletions(-) diff --git a/src/java/com/netflix/ice/basic/BasicReservationService.java b/src/java/com/netflix/ice/basic/BasicReservationService.java index 54f828a5..4f06ee97 100644 --- a/src/java/com/netflix/ice/basic/BasicReservationService.java +++ b/src/java/com/netflix/ice/basic/BasicReservationService.java @@ -17,7 +17,8 @@ */ package com.netflix.ice.basic; -import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.*; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -25,7 +26,8 @@ import com.netflix.ice.common.Poller; import com.netflix.ice.common.TagGroup; import com.netflix.ice.processor.Ec2InstanceReservationPrice; -import com.netflix.ice.processor.Ec2InstanceReservationPrice.*; +import com.netflix.ice.processor.Ec2InstanceReservationPrice.ReservationPeriod; +import com.netflix.ice.processor.Ec2InstanceReservationPrice.ReservationUtilization; import com.netflix.ice.processor.ProcessorConfig; import com.netflix.ice.processor.ReservationService; import com.netflix.ice.tag.*; @@ -36,7 +38,9 @@ import org.slf4j.LoggerFactory; import java.io.*; -import java.util.*; +import java.util.Collection; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentSkipListMap; public class BasicReservationService extends Poller implements ReservationService { @@ -142,7 +146,10 @@ private void pollAPI() throws Exception { .withFilters(new com.amazonaws.services.ec2.model.Filter().withName("marketplace").withValues("false")); String token = null; boolean hasNewPrice = false; - AmazonEC2Client ec2Client = new AmazonEC2Client(AwsUtils.awsCredentialsProvider, AwsUtils.clientConfig); + AmazonEC2ClientBuilder amazonEC2ClientBuilder = AmazonEC2ClientBuilder + .standard() + .withCredentials(AwsUtils.awsCredentialsProvider) + .withClientConfiguration(AwsUtils.clientConfig); for (Region region: Region.getAllRegions()) { // GovCloud uses different credentials than standard AWS, so you would need two separate @@ -151,7 +158,8 @@ private void pollAPI() throws Exception { if (region == Region.US_GOV_WEST_1) { continue; } - ec2Client.setEndpoint("ec2." + region.name + ".amazonaws.com"); + amazonEC2ClientBuilder.setRegion(region.name); + AmazonEC2 ec2Client = amazonEC2ClientBuilder.build(); do { if (!StringUtils.isEmpty(token)) req.setNextToken(token); @@ -159,7 +167,7 @@ private void pollAPI() throws Exception { token = offers.getNextToken(); for (ReservedInstancesOffering offer: offers.getReservedInstancesOfferings()) { - if (offer.getProductDescription().indexOf("Amazon VPC") >= 0) + if (offer.getProductDescription().contains("Amazon VPC")) continue; // Ignore Region-Wide RIs @@ -193,9 +201,9 @@ private void pollAPI() throws Exception { } } } while (!StringUtils.isEmpty(token)); + ec2Client.shutdown(); } - ec2Client.shutdown(); if (hasNewPrice) { for (ReservationUtilization utilization: files.keySet()) { File file = files.get(utilization); diff --git a/src/java/com/netflix/ice/processor/ReservationCapacityPoller.java b/src/java/com/netflix/ice/processor/ReservationCapacityPoller.java index f07968f0..27705a3a 100644 --- a/src/java/com/netflix/ice/processor/ReservationCapacityPoller.java +++ b/src/java/com/netflix/ice/processor/ReservationCapacityPoller.java @@ -17,14 +17,12 @@ */ package com.netflix.ice.processor; -import com.amazonaws.auth.AWSCredentialsProvider; import com.amazonaws.auth.AWSSessionCredentials; -import com.amazonaws.services.ec2.AmazonEC2Client; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.services.ec2.AmazonEC2; +import com.amazonaws.services.ec2.AmazonEC2ClientBuilder; import com.amazonaws.services.ec2.model.DescribeReservedInstancesResult; import com.amazonaws.services.ec2.model.ReservedInstances; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; -import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; -import com.amazonaws.services.securitytoken.model.AssumeRoleResult; import com.amazonaws.services.securitytoken.model.Credentials; import com.google.common.collect.Maps; import com.netflix.ice.common.AwsUtils; @@ -116,12 +114,12 @@ protected void poll() throws Exception { for (Account account: config.accountService.getReservationAccounts().keySet()) { try { - AmazonEC2Client ec2Client; + AmazonEC2ClientBuilder amazonEC2ClientBuilder = AmazonEC2ClientBuilder.standard(); String assumeRole = config.accountService.getReservationAccessRoles().get(account); if (assumeRole != null) { String externalId = config.accountService.getReservationAccessExternalIds().get(account); final Credentials credentials = AwsUtils.getAssumedCredentials(account.id, assumeRole, externalId); - ec2Client = new AmazonEC2Client(new AWSSessionCredentials() { + AWSSessionCredentials awsCredentials = new AWSSessionCredentials() { public String getAWSAccessKeyId() { return credentials.getAccessKeyId(); } @@ -133,11 +131,14 @@ public String getAWSSecretKey() { public String getSessionToken() { return credentials.getSessionToken(); } - }); + }; + amazonEC2ClientBuilder.withCredentials(new AWSStaticCredentialsProvider(awsCredentials)); + } + else { + amazonEC2ClientBuilder + .withClientConfiguration(AwsUtils.clientConfig) + .withCredentials(AwsUtils.awsCredentialsProvider); } - else - ec2Client = new AmazonEC2Client(AwsUtils.awsCredentialsProvider.getCredentials(), AwsUtils.clientConfig); - for (Region region: Region.getAllRegions()) { // GovCloud uses different credentials than standard AWS, so you would need two separate // sets of credentials if you wanted to poll for RIs in both environments. For now, we @@ -145,8 +146,8 @@ public String getSessionToken() { if (region == Region.US_GOV_WEST_1) { continue; } - - ec2Client.setEndpoint("ec2." + region.name + ".amazonaws.com"); + amazonEC2ClientBuilder.setRegion(region.name); + AmazonEC2 ec2Client = amazonEC2ClientBuilder.build(); try { DescribeReservedInstancesResult result = ec2Client.describeReservedInstances(); @@ -164,9 +165,8 @@ public String getSessionToken() { catch (Exception e) { logger.error("error in describeReservedInstances for " + region.name + " " + account.name, e); } + ec2Client.shutdown(); } - - ec2Client.shutdown(); } catch (Exception e) { logger.error("Error in describeReservedInstances for " + account.name, e); From 9aa28698f84ffc19ee627d2f696878f78eaab210 Mon Sep 17 00:00:00 2001 From: wei Date: Thu, 2 Nov 2017 16:23:16 -0700 Subject: [PATCH 3/3] Update deprecated AWS client --- .../basic/BasicWeeklyCostEmailService.java | 11 +++-- src/java/com/netflix/ice/common/AwsUtils.java | 44 ++++++++++++------- .../ice/processor/BillingFileProcessor.java | 4 +- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/src/java/com/netflix/ice/basic/BasicWeeklyCostEmailService.java b/src/java/com/netflix/ice/basic/BasicWeeklyCostEmailService.java index 4cea27c4..fe61b8a7 100644 --- a/src/java/com/netflix/ice/basic/BasicWeeklyCostEmailService.java +++ b/src/java/com/netflix/ice/basic/BasicWeeklyCostEmailService.java @@ -17,7 +17,7 @@ */ package com.netflix.ice.basic; -import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService; import com.amazonaws.services.simpleemail.model.RawMessage; import com.amazonaws.services.simpleemail.model.SendRawEmailRequest; import com.google.common.collect.Lists; @@ -67,7 +67,10 @@ import java.text.FieldPosition; import java.text.NumberFormat; import java.text.ParsePosition; -import java.util.*; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; public class BasicWeeklyCostEmailService extends Poller { @@ -176,7 +179,7 @@ public synchronized void trigger(boolean test) { try { headerNote = getHeaderNote(); throughputMetrics = getThroughputMetrics(); - AmazonSimpleEmailServiceClient emailService = AwsUtils.getAmazonSimpleEmailServiceClient(); + AmazonSimpleEmailService emailService = AwsUtils.getAmazonSimpleEmailServiceClient(); Map appgroups = applicationGroupService.getApplicationGroups(); Map> appgroupsByEmail = collectEmails(appgroups); @@ -440,7 +443,7 @@ private MimeBodyPart constructEmail(int index, ApplicationGroup appGroup, String return mimeBodyPart; } - private void sendEmail(boolean test, AmazonSimpleEmailServiceClient emailService, String email, List appGroups) + private void sendEmail(boolean test, AmazonSimpleEmailService emailService, String email, List appGroups) throws IOException, MessagingException { StringBuilder body = new StringBuilder(); diff --git a/src/java/com/netflix/ice/common/AwsUtils.java b/src/java/com/netflix/ice/common/AwsUtils.java index 9b7cb721..68f686a2 100755 --- a/src/java/com/netflix/ice/common/AwsUtils.java +++ b/src/java/com/netflix/ice/common/AwsUtils.java @@ -24,12 +24,16 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.*; -import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient; +import com.amazonaws.services.securitytoken.AWSSecurityTokenService; +import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; import com.amazonaws.services.securitytoken.model.AssumeRoleResult; import com.amazonaws.services.securitytoken.model.Credentials; +import com.amazonaws.services.simpledb.AmazonSimpleDB; import com.amazonaws.services.simpledb.AmazonSimpleDBClient; -import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.amazonaws.services.simpledb.AmazonSimpleDBClientBuilder; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClientBuilder; import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.commons.lang.StringUtils; @@ -60,9 +64,9 @@ public class AwsUtils { public static long hourMillis = 3600000L; private static AmazonS3 s3Client; - private static AmazonSimpleEmailServiceClient emailServiceClient; - private static AmazonSimpleDBClient simpleDBClient; - private static AWSSecurityTokenServiceClient securityClient; + private static AmazonSimpleEmailService emailServiceClient; + private static AmazonSimpleDB simpleDBClient; + private static AWSSecurityTokenService securityClient; public static AWSCredentialsProvider awsCredentialsProvider; public static ClientConfiguration clientConfig; @@ -103,30 +107,36 @@ public static void init(AWSCredentialsProvider credentialsProvider) { amazonS3ClientBuilder.withRegion(System.getProperty("EC2_REGION")); } s3Client = amazonS3ClientBuilder.build(); - securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig); + securityClient = AWSSecurityTokenServiceClientBuilder + .standard().withCredentials(awsCredentialsProvider) + .withClientConfiguration(clientConfig) + .build(); } public static AmazonS3 getAmazonS3Client() { return s3Client; } - public static AmazonSimpleEmailServiceClient getAmazonSimpleEmailServiceClient() { + public static AmazonSimpleEmailService getAmazonSimpleEmailServiceClient() { if (emailServiceClient == null) - emailServiceClient = new AmazonSimpleEmailServiceClient(awsCredentialsProvider, clientConfig); + emailServiceClient = AmazonSimpleEmailServiceClientBuilder.standard() + .withCredentials(awsCredentialsProvider) + .withClientConfiguration(clientConfig) + .build(); return emailServiceClient; } - public static AmazonSimpleDBClient getAmazonSimpleDBClient() { + public static AmazonSimpleDB getAmazonSimpleDBClient() { if (simpleDBClient == null) { - simpleDBClient = new AmazonSimpleDBClient(awsCredentialsProvider, clientConfig); - if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) { - if ("global".equals(System.getProperty("EC2_REGION"))) { - simpleDBClient.setEndpoint("sdb.amazonaws.com"); - } - else { - simpleDBClient.setEndpoint("sdb." + System.getProperty("EC2_REGION") + ".amazonaws.com"); - } + AmazonSimpleDBClientBuilder amazonSimpleDBClientBuilder = AmazonSimpleDBClient.builder() + .withCredentials(awsCredentialsProvider) + .withClientConfiguration(clientConfig); + + if (System.getProperty("EC2_REGION") != null) { + amazonSimpleDBClientBuilder.setRegion(System.getProperty("EC2_REGION")); } + simpleDBClient = amazonSimpleDBClientBuilder + .build(); } return simpleDBClient; } diff --git a/src/java/com/netflix/ice/processor/BillingFileProcessor.java b/src/java/com/netflix/ice/processor/BillingFileProcessor.java index 2593490b..4a1929fe 100644 --- a/src/java/com/netflix/ice/processor/BillingFileProcessor.java +++ b/src/java/com/netflix/ice/processor/BillingFileProcessor.java @@ -20,7 +20,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient; +import com.amazonaws.services.simpleemail.AmazonSimpleEmailService; import com.amazonaws.services.simpleemail.model.*; import com.csvreader.CsvReader; import com.google.common.collect.Lists; @@ -729,7 +729,7 @@ endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24) request.withDestination(new Destination(emails)); request.withMessage(new Message(new Content(subject), new Body().withHtml(new Content(body.toString())))); - AmazonSimpleEmailServiceClient emailService = AwsUtils.getAmazonSimpleEmailServiceClient(); + AmazonSimpleEmailService emailService = AwsUtils.getAmazonSimpleEmailServiceClient(); try { emailService.sendEmail(request); updateLastAlertMillis(endMilli);