Update AWS Client

.gitignore

@@ -12,6 +12,7 @@
 /*.tmproj
 /ivy*
 /eclipse
+/.idea/*
 
 # default HSQL database files for production mode
 /prodDb.*

grails-app/conf/BuildConfig.groovy

@@ -66,12 +66,12 @@ grails.project.dependency.resolution = {
 
         compile(
                 // Amazon Web Services programmatic interface
-                'com.amazonaws:aws-java-sdk:1.11.136',
+                'com.amazonaws:aws-java-sdk:1.11.221',
                 // Transitive dependencies of aws-java-sdk, but also used directly.
                 // It would be great if we could upgrade httpcore and httpclient, but we can't until the AWS Java SDK
                 // upgrades its dependencies. If we simply upgrade these, then some Amazon calls fail.
-                'org.apache.httpcomponents:httpcore:4.4.4',
-                'org.apache.httpcomponents:httpclient:4.5.2',
+                'org.apache.httpcomponents:httpcore:4.4.7',
+                'org.apache.httpcomponents:httpclient:4.5.3',
 
                 // Explicitly including aws-java-sdk transitive dependencies
                 'org.codehaus.jackson:jackson-core-asl:1.8.9',

src/java/com/netflix/ice/basic/BasicManagers.java

@@ -17,13 +17,18 @@
  */
 package com.netflix.ice.basic;
 
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-import com.netflix.ice.common.*;
+import com.netflix.ice.common.AwsUtils;
+import com.netflix.ice.common.ConsolidateType;
+import com.netflix.ice.common.Poller;
 import com.netflix.ice.processor.TagGroupWriter;
-import com.netflix.ice.reader.*;
+import com.netflix.ice.reader.DataManager;
+import com.netflix.ice.reader.Managers;
+import com.netflix.ice.reader.ReaderConfig;
+import com.netflix.ice.reader.TagGroupManager;
 import com.netflix.ice.tag.Product;
 import com.netflix.ice.tag.Tag;
 
@@ -91,7 +96,7 @@ private void doWork() {
         TreeMap<Key, BasicDataManager> usageManagers = Maps.newTreeMap(this.usageManagers);
 
         Set<Product> newProducts = Sets.newHashSet();
-        AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
+        AmazonS3 s3Client = AwsUtils.getAmazonS3Client();
         for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).getObjectSummaries()) {
             String key = s3ObjectSummary.getKey();
             Product product;

src/java/com/netflix/ice/basic/BasicReservationService.java

@@ -17,15 +17,17 @@
  */
 package com.netflix.ice.basic;
 
-import com.amazonaws.services.ec2.AmazonEC2Client;
+import com.amazonaws.services.ec2.AmazonEC2;
+import com.amazonaws.services.ec2.AmazonEC2ClientBuilder;
 import com.amazonaws.services.ec2.model.*;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.netflix.ice.common.AwsUtils;
 import com.netflix.ice.common.Poller;
 import com.netflix.ice.common.TagGroup;
 import com.netflix.ice.processor.Ec2InstanceReservationPrice;
-import com.netflix.ice.processor.Ec2InstanceReservationPrice.*;
+import com.netflix.ice.processor.Ec2InstanceReservationPrice.ReservationPeriod;
+import com.netflix.ice.processor.Ec2InstanceReservationPrice.ReservationUtilization;
 import com.netflix.ice.processor.ProcessorConfig;
 import com.netflix.ice.processor.ReservationService;
 import com.netflix.ice.tag.*;
@@ -36,7 +38,9 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
-import java.util.*;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentSkipListMap;
 
 public class BasicReservationService extends Poller implements ReservationService {
@@ -142,7 +146,10 @@ private void pollAPI() throws Exception {
                 .withFilters(new com.amazonaws.services.ec2.model.Filter().withName("marketplace").withValues("false"));
         String token = null;
         boolean hasNewPrice = false;
-        AmazonEC2Client ec2Client = new AmazonEC2Client(AwsUtils.awsCredentialsProvider, AwsUtils.clientConfig);
+        AmazonEC2ClientBuilder amazonEC2ClientBuilder = AmazonEC2ClientBuilder
+                .standard()
+                .withCredentials(AwsUtils.awsCredentialsProvider)
+                .withClientConfiguration(AwsUtils.clientConfig);
 
         for (Region region: Region.getAllRegions()) {
             // GovCloud uses different credentials than standard AWS, so you would need two separate
@@ -151,15 +158,16 @@ private void pollAPI() throws Exception {
             if (region == Region.US_GOV_WEST_1) {
                 continue;
             }
-            ec2Client.setEndpoint("ec2." + region.name + ".amazonaws.com");
+            amazonEC2ClientBuilder.setRegion(region.name);
+            AmazonEC2 ec2Client = amazonEC2ClientBuilder.build();
             do {
                 if (!StringUtils.isEmpty(token))
                     req.setNextToken(token);
                 DescribeReservedInstancesOfferingsResult offers = ec2Client.describeReservedInstancesOfferings(req);
                 token = offers.getNextToken();
 
                 for (ReservedInstancesOffering offer: offers.getReservedInstancesOfferings()) {
-                    if (offer.getProductDescription().indexOf("Amazon VPC") >= 0)
+                    if (offer.getProductDescription().contains("Amazon VPC"))
                         continue;
 
                     // Ignore Region-Wide RIs
@@ -193,9 +201,9 @@ private void pollAPI() throws Exception {
                     }
                 }
             } while (!StringUtils.isEmpty(token));
+            ec2Client.shutdown();
         }
 
-        ec2Client.shutdown();
         if (hasNewPrice) {
             for (ReservationUtilization utilization: files.keySet()) {
                 File file = files.get(utilization);

src/java/com/netflix/ice/basic/BasicS3ApplicationGroupService.java

@@ -17,7 +17,7 @@
  */
 package com.netflix.ice.basic;
 
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.google.common.collect.Maps;
 import com.netflix.ice.common.AwsUtils;
@@ -38,7 +38,7 @@
 
 public class BasicS3ApplicationGroupService implements ApplicationGroupService {
     private final static Logger logger = LoggerFactory.getLogger(BasicS3ApplicationGroupService.class);
-    private AmazonS3Client s3Client;
+    private AmazonS3 s3Client;
     private ReaderConfig config;
 
     public void init() {

src/java/com/netflix/ice/basic/BasicWeeklyCostEmailService.java

@@ -17,7 +17,7 @@
  */
 package com.netflix.ice.basic;
 
-import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;
+import com.amazonaws.services.simpleemail.AmazonSimpleEmailService;
 import com.amazonaws.services.simpleemail.model.RawMessage;
 import com.amazonaws.services.simpleemail.model.SendRawEmailRequest;
 import com.google.common.collect.Lists;
@@ -67,7 +67,10 @@
 import java.text.FieldPosition;
 import java.text.NumberFormat;
 import java.text.ParsePosition;
-import java.util.*;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Properties;
 
 public class BasicWeeklyCostEmailService extends Poller {
 
@@ -176,7 +179,7 @@ public synchronized void trigger(boolean test) {
         try {
             headerNote = getHeaderNote();
             throughputMetrics = getThroughputMetrics();
-            AmazonSimpleEmailServiceClient emailService = AwsUtils.getAmazonSimpleEmailServiceClient();
+            AmazonSimpleEmailService emailService = AwsUtils.getAmazonSimpleEmailServiceClient();
             Map<String, ApplicationGroup> appgroups = applicationGroupService.getApplicationGroups();
             Map<String, List<ApplicationGroup>> appgroupsByEmail = collectEmails(appgroups);
 
@@ -440,7 +443,7 @@ private MimeBodyPart constructEmail(int index, ApplicationGroup appGroup, String
         return mimeBodyPart;
     }
 
-    private void sendEmail(boolean test, AmazonSimpleEmailServiceClient emailService, String email, List<ApplicationGroup> appGroups)
+    private void sendEmail(boolean test, AmazonSimpleEmailService emailService, String email, List<ApplicationGroup> appGroups)
         throws IOException, MessagingException {
 
         StringBuilder body = new StringBuilder();

src/java/com/netflix/ice/basic/MapDb.java

@@ -17,7 +17,7 @@
  */
 package com.netflix.ice.basic;
 
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.netflix.ice.common.AwsUtils;
 import com.netflix.ice.processor.ProcessorConfig;
@@ -47,7 +47,7 @@ public class MapDb {
         this.dbName = "db_" + name;
         File file = new File(config.localDir, dbName);
         if (!file.exists()) {
-            AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
+            AmazonS3 s3Client = AwsUtils.getAmazonS3Client();
             for (S3ObjectSummary s3ObjectSummary: s3Client.listObjects(config.workS3BucketName, config.workS3BucketPrefix + this.dbName).getObjectSummaries()) {
                 File dbFile = new File(config.localDir, s3ObjectSummary.getKey().substring(config.workS3BucketPrefix.length()));
                 AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, dbFile);
@@ -103,7 +103,7 @@ void commit() {
     }
 
     void upload() {
-        AmazonS3Client s3Client = AwsUtils.getAmazonS3Client();
+        AmazonS3 s3Client = AwsUtils.getAmazonS3Client();
 
         File dir = new File(config.localDir);
         File[] files = dir.listFiles(new FilenameFilter() {

src/java/com/netflix/ice/common/AwsUtils.java

@@ -17,17 +17,23 @@
  */
 package com.netflix.ice.common;
 
+import com.amazonaws.ClientConfiguration;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicSessionCredentials;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.*;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
 import com.amazonaws.services.securitytoken.model.AssumeRoleRequest;
 import com.amazonaws.services.securitytoken.model.AssumeRoleResult;
 import com.amazonaws.services.securitytoken.model.Credentials;
+import com.amazonaws.services.simpledb.AmazonSimpleDB;
 import com.amazonaws.services.simpledb.AmazonSimpleDBClient;
-import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClient;
-import com.amazonaws.ClientConfiguration;
+import com.amazonaws.services.simpledb.AmazonSimpleDBClientBuilder;
+import com.amazonaws.services.simpleemail.AmazonSimpleEmailService;
+import com.amazonaws.services.simpleemail.AmazonSimpleEmailServiceClientBuilder;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Sets;
 import org.apache.commons.lang.StringUtils;
@@ -57,10 +63,10 @@ public class AwsUtils {
     public static final DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HHa").withZone(DateTimeZone.UTC);
     public static long hourMillis = 3600000L;
 
-    private static AmazonS3Client s3Client;
-    private static AmazonSimpleEmailServiceClient emailServiceClient;
-    private static AmazonSimpleDBClient simpleDBClient;
-    private static AWSSecurityTokenServiceClient securityClient;
+    private static AmazonS3 s3Client;
+    private static AmazonSimpleEmailService emailServiceClient;
+    private static AmazonSimpleDB simpleDBClient;
+    private static AWSSecurityTokenService securityClient;
     public static AWSCredentialsProvider awsCredentialsProvider;
     public static ClientConfiguration clientConfig;
 
@@ -89,43 +95,48 @@ public static void init(AWSCredentialsProvider credentialsProvider) {
         clientConfig = new ClientConfiguration();
         String proxyHost = System.getProperty("https.proxyHost");
         String proxyPort = System.getProperty("https.proxyPort");
-        if(proxyHost != null && proxyPort != null) {
+        if (proxyHost != null && proxyPort != null) {
             clientConfig.setProxyHost(proxyHost);
             clientConfig.setProxyPort(Integer.parseInt(proxyPort));
         }
-        s3Client = new AmazonS3Client(awsCredentialsProvider, clientConfig);
-        securityClient = new AWSSecurityTokenServiceClient(awsCredentialsProvider, clientConfig);
-        if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) {
-            if ("global".equals(System.getProperty("EC2_REGION"))) {
-                s3Client.setEndpoint("s3.amazonaws.com");
-            }
-            else {
-                s3Client.setEndpoint("s3-" + System.getProperty("EC2_REGION") + ".amazonaws.com");
-            }
+        AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder
+                .standard()
+                .withCredentials(credentialsProvider)
+                .withClientConfiguration(clientConfig);
+        if (System.getProperty("EC2_REGION") != null) {
+            amazonS3ClientBuilder.withRegion(System.getProperty("EC2_REGION"));
         }
+        s3Client = amazonS3ClientBuilder.build();
+        securityClient = AWSSecurityTokenServiceClientBuilder
+                .standard().withCredentials(awsCredentialsProvider)
+                .withClientConfiguration(clientConfig)
+                .build();
     }
 
-    public static AmazonS3Client getAmazonS3Client() {
+    public static AmazonS3 getAmazonS3Client() {
         return s3Client;
     }
 
-    public static AmazonSimpleEmailServiceClient getAmazonSimpleEmailServiceClient() {
+    public static AmazonSimpleEmailService getAmazonSimpleEmailServiceClient() {
         if (emailServiceClient == null)
-            emailServiceClient = new AmazonSimpleEmailServiceClient(awsCredentialsProvider, clientConfig);
+            emailServiceClient = AmazonSimpleEmailServiceClientBuilder.standard()
+                    .withCredentials(awsCredentialsProvider)
+                    .withClientConfiguration(clientConfig)
+                    .build();
         return emailServiceClient;
     }
 
-    public static AmazonSimpleDBClient getAmazonSimpleDBClient() {
+    public static AmazonSimpleDB getAmazonSimpleDBClient() {
         if (simpleDBClient == null) {
-            simpleDBClient = new AmazonSimpleDBClient(awsCredentialsProvider, clientConfig);
-            if (System.getProperty("EC2_REGION") != null && !"us-east-1".equals(System.getProperty("EC2_REGION"))) {
-                if ("global".equals(System.getProperty("EC2_REGION"))) {
-                    simpleDBClient.setEndpoint("sdb.amazonaws.com");
-                }
-                else {
-                    simpleDBClient.setEndpoint("sdb." + System.getProperty("EC2_REGION") + ".amazonaws.com");
-                }
+            AmazonSimpleDBClientBuilder amazonSimpleDBClientBuilder = AmazonSimpleDBClient.builder()
+                    .withCredentials(awsCredentialsProvider)
+                    .withClientConfiguration(clientConfig);
+
+            if (System.getProperty("EC2_REGION") != null) {
+                amazonSimpleDBClientBuilder.setRegion(System.getProperty("EC2_REGION"));
             }
+            simpleDBClient = amazonSimpleDBClientBuilder
+                    .build();
         }
         return simpleDBClient;
     }
@@ -158,19 +169,21 @@ public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix)
      */
     public static List<S3ObjectSummary> listAllObjects(String bucket, String prefix, String accountId,
                                                        String assumeRole, String externalId) {
-        AmazonS3Client s3Client = AwsUtils.s3Client;
+        AmazonS3 s3Client = AwsUtils.s3Client;
 
         try {
             ListObjectsRequest request = new ListObjectsRequest().withBucketName(bucket).withPrefix(prefix);
             List<S3ObjectSummary> result = Lists.newLinkedList();
 
             if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
                 Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
-                s3Client = new AmazonS3Client(
-                        new BasicSessionCredentials(assumedCredentials.getAccessKeyId(),
-                                assumedCredentials.getSecretAccessKey(),
-                                assumedCredentials.getSessionToken()),
-                                clientConfig);
+
+                BasicAWSCredentials awsCreds = new BasicAWSCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey());
+                AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder
+                        .standard()
+                        .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
+                        .withClientConfiguration(clientConfig);
+                s3Client = amazonS3ClientBuilder.build();
             }
 
             ObjectListing page = null;
@@ -260,20 +273,20 @@ public static long getLastModified(String bucketName, String fileKey) {
 
     public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file,
                                                      long milles, String accountId, String assumeRole, String externalId) {
-        AmazonS3Client s3Client = AwsUtils.s3Client;
+        AmazonS3 s3Client = AwsUtils.s3Client;
 
         try {
             if (!StringUtils.isEmpty(accountId) && !StringUtils.isEmpty(assumeRole)) {
                 Credentials assumedCredentials = getAssumedCredentials(accountId, assumeRole, externalId);
-                s3Client = new AmazonS3Client(
-                        new BasicSessionCredentials(assumedCredentials.getAccessKeyId(),
-                                assumedCredentials.getSecretAccessKey(),
-                                assumedCredentials.getSessionToken()),
-                                clientConfig);
-            }
-
-            if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
-                s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com");
+                BasicAWSCredentials awsCreds = new BasicAWSCredentials(assumedCredentials.getAccessKeyId(), assumedCredentials.getSecretAccessKey());
+                AmazonS3ClientBuilder amazonS3ClientBuilder = AmazonS3ClientBuilder
+                        .standard()
+                        .withCredentials(new AWSStaticCredentialsProvider(awsCreds))
+                        .withClientConfiguration(clientConfig);
+                if (bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
+                    amazonS3ClientBuilder.setRegion(bucketFileRegion);
+                }
+                s3Client = amazonS3ClientBuilder.build();
             }
 
             ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
@@ -333,7 +346,7 @@ private static boolean download(String bucketName, String fileKey, File file) {
         return download(s3Client, bucketName, fileKey, file);
     }
 
-    private static boolean download(AmazonS3Client s3Client, String bucketName, String fileKey, File file) {
+    private static boolean download(AmazonS3 s3Client, String bucketName, String fileKey, File file) {
         do {
             S3Object s3Object = s3Client.getObject(bucketName, fileKey);
             InputStream input = s3Object.getObjectContent();