Some smaller improvements on error logs/exceptions, ignores, file-checks, ...

.gitignore

@@ -0,0 +1,51 @@
+# .gitignore for Grails 1.2 and 1.3
+
+# web application files
+/web-app/WEB-INF
+
+# IDE support files
+/.launch
+/.settings
+/*.launch
+/*.tmproj
+/ivy*
+/eclipse
+/bin
+
+# default HSQL database files for production mode
+/prodDb.*
+
+# general HSQL database files
+*Db.properties
+*Db.script
+
+# logs
+/stacktrace.log
+/test/reports
+/logs
+
+# project release file
+/*.war
+
+# plugin release file
+/*.zip
+
+# older plugin install locations
+/plugins
+/web-app/plugins
+/web-app/WEB-INF/classes
+
+# "temporary" build files
+/target
+
+# other
+*.iws
+
+# local data
+/ice_processor
+/ice_reader
+/work
+/test
+
+# sensitive data not checked in
+account.properties

src/java/com/netflix/ice/basic/BasicLineItemProcessor.java

@@ -97,8 +97,10 @@ public Result process(long startMilli, boolean processDelayed, ProcessorConfig c
             return Result.ignore;
 
         Account account = config.accountService.getAccountById(items[accountIdIndex]);
-        if (account == null)
-            return Result.ignore;
+        if (account == null) {
+        	logger.warn("Could not find account: " + items[accountIdIndex] + " in AccountService");
+        	return Result.ignore;
+        }
 
         double usageValue = Double.parseDouble(items[usageQuantityIndex]);
         double costValue = Double.parseDouble(items[costIndex]);

src/java/com/netflix/ice/basic/BasicReservationService.java

@@ -106,8 +106,7 @@ public void init() {
                 pollAPI();
             }
             catch (Exception e) {
-                logger.error("failed to poll reservation prices", e);
-                throw new RuntimeException("failed to poll reservation prices for " + e.getMessage());
+                throw new RuntimeException("failed to poll reservation prices", e);
             }
         }
         else {
@@ -318,6 +317,10 @@ public double getLatestHourlyTotalPrice(
         Ec2InstanceReservationPrice ec2Price =
             ec2InstanceReservationPrices.get(utilization).get(new Ec2InstanceReservationPrice.Key(region, usageType));
 
+        if(ec2Price == null) {
+        	throw new IllegalStateException("Did not find an EC2 price for region " + region + ", type " + usageType + " and utilization " + utilization);
+        }
+
         double tier = getEc2Tier(time);
         return ec2Price.hourlyPrice.getPrice(null).getPrice(tier) +
                ec2Price.upfrontPrice.getPrice(null).getUpfrontAmortized(time, term, tier);
@@ -367,6 +370,10 @@ public ReservationInfo getReservation(
             houlyCost = houlyCost / count;
         }
 
+        if(houlyCost == 0) {
+        	throw new IllegalStateException("Did not find an EC2 price for region " + tagGroup.region + ", type " + tagGroup.usageType + " and utilization " + utilization);
+        }
+
         return new ReservationInfo(count, upfrontAmortized, houlyCost);
     }
 

src/java/com/netflix/ice/common/Config.java

@@ -48,8 +48,8 @@ public Config(
             AccountService accountService,
             ProductService productService,
             ResourceService resourceService) {
+    	if (properties == null) throw new IllegalArgumentException("properties must be specified");
         if (properties.getProperty(IceOptions.START_MILLIS) == null) throw new IllegalArgumentException("IceOptions.START_MILLIS must be specified");
-        if (properties == null) throw new IllegalArgumentException("properties must be specified");
         if (credentialsProvider == null) throw new IllegalArgumentException("credentialsProvider must be specified");
         if (accountService == null) throw new IllegalArgumentException("accountService must be specified");
         if (productService == null) throw new IllegalArgumentException("productService must be specified");

src/java/com/netflix/ice/processor/BillingFileProcessor.java

@@ -560,7 +560,7 @@ private void processBillingZipFile(File file, boolean withTags) throws IOExcepti
 
     private void processBillingFile(String fileName, InputStream tempIn, boolean withTags) {
 
-        CsvReader reader = new CsvReader(new InputStreamReader(tempIn), ',');
+        CsvReader reader = new CsvReader(new BufferedReader(new InputStreamReader(tempIn), 1024*1024), ',');
 
         long lineNumber = 0;
         List<String[]> delayedItems = Lists.newArrayList();

src/java/com/netflix/ice/processor/DataWriter.java

@@ -44,11 +44,14 @@ public class DataWriter {
             AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file);
         }
 
-        if (file.exists()) {
+        if (file.exists() && file.length() > 0) {
             DataInputStream in = new DataInputStream(new FileInputStream(file));
             try {
                 data = ReadWriteData.Serializer.deserialize(in);
             }
+            catch (EOFException e) {
+            	throw new IllegalStateException("While handling file: " + file, e);
+            }
             finally {
                 in.close();
             }

src/java/com/netflix/ice/processor/TagGroupWriter.java

@@ -46,7 +46,12 @@ public class TagGroupWriter {
         if (file.exists()) {
             DataInputStream in = new DataInputStream(new FileInputStream(file));
             try {
-                tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in);
+            	try {
+            		tagGroups = TagGroup.Serializer.deserializeTagGroups(config, in);
+            	} catch (EOFException e) {
+            		// how do we handle a corrupted file!
+            		throw new IllegalStateException("While handling file " + file, e);
+            	}
             }
             finally {
                 if (in != null)
@@ -69,7 +74,7 @@ void archive(Long monthMilli,Collection<TagGroup> tagGroups) throws IOException
             out.close();
         }
 
-        logger.info(dbName + " uploading to s3...");
+        logger.info(dbName + " uploading to s3 bucket " + config.workS3BucketName + " with prefix '" + config.workS3BucketPrefix + "'...");
         AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, dbName);
         logger.info(dbName + " uploading done.");
     }