Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added region property for billing buckets to reference properly if v4… #193

Merged
merged 1 commit into from
Jun 7, 2017
Merged

Added region property for billing buckets to reference properly if v4… #193

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions grails-app/conf/BootStrap.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ class BootStrap {

properties.setProperty(IceOptions.LOCAL_DIR, prop.getProperty("ice.processor.localDir"));
properties.setProperty(IceOptions.BILLING_S3_BUCKET_NAME, prop.getProperty(IceOptions.BILLING_S3_BUCKET_NAME));
properties.setProperty(IceOptions.BILLING_S3_BUCKET_REGION, prop.getProperty(IceOptions.BILLING_S3_BUCKET_REGION));
properties.setProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, prop.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, ""));
properties.setProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, prop.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, ""));
properties.setProperty(IceOptions.BILLING_ACCESS_ROLENAME, prop.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, ""));
Expand Down
8 changes: 6 additions & 2 deletions src/java/com/netflix/ice/common/AwsUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,8 +258,8 @@ public static long getLastModified(String bucketName, String fileKey) {
}
}

public static boolean downloadFileIfChangedSince(String bucketName, String bucketFilePrefix, File file, long milles, String accountId,
String assumeRole, String externalId) {
public static boolean downloadFileIfChangedSince(String bucketName, String bucketFileRegion, String bucketFilePrefix, File file,
long milles, String accountId, String assumeRole, String externalId) {
AmazonS3Client s3Client = AwsUtils.s3Client;

try {
Expand All @@ -272,6 +272,10 @@ public static boolean downloadFileIfChangedSince(String bucketName, String bucke
clientConfig);
}

if(bucketFileRegion != null && !bucketFileRegion.isEmpty()) {
s3Client.setEndpoint("s3-" + bucketFileRegion + ".amazonaws.com");
}

ObjectMetadata metadata = s3Client.getObjectMetadata(bucketName, bucketFilePrefix + file.getName());
boolean download = !file.exists() || metadata.getLastModified().getTime() > milles;

Expand Down
6 changes: 6 additions & 0 deletions src/java/com/netflix/ice/common/IceOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,12 @@ public class IceOptions {
*/
public static final String BILLING_S3_BUCKET_NAME = "ice.billing_s3bucketname";

/**
* Region for billing s3 bucket. It should be specified for buckets using v4 validation ",".
* It must be specified in Config.
*/
public static final String BILLING_S3_BUCKET_REGION = "ice.billing_s3bucketregion";

/**
* Prefix of billing files in billing s3 bucket. For multiple payer accounts, multiple bucket prefixes can be specified delimited by comma ",".
* It must be specified in Config.
Expand Down
15 changes: 9 additions & 6 deletions src/java/com/netflix/ice/processor/BillingFileProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ protected void poll() throws Exception {
// list the tar.gz file in billing file folder
for (int i = 0; i < config.billingS3BucketNames.length; i++) {
String billingS3BucketName = config.billingS3BucketNames[i];
String billingS3BucketRegion = config.billingS3BucketRegions.length > i ? config.billingS3BucketRegions[i] : "";
String billingS3BucketPrefix = config.billingS3BucketPrefixes.length > i ? config.billingS3BucketPrefixes[i] : "";
String accountId = config.billingAccountIds.length > i ? config.billingAccountIds[i] : "";
String billingAccessRoleName = config.billingAccessRoleNames.length > i ? config.billingAccessRoleNames[i] : "";
Expand Down Expand Up @@ -128,7 +129,7 @@ protected void poll() throws Exception {
list = Lists.newArrayList();
filesToProcess.put(key, list);
}
list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
list.add(new BillingFile(filesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
}

for (DateTime key: monitorFilesToProcessInOneBucket.keySet()) {
Expand All @@ -137,7 +138,7 @@ protected void poll() throws Exception {
list = Lists.newArrayList();
monitorFilesToProcess.put(key, list);
}
list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix));
list.add(new BillingFile(monitorFilesToProcessInOneBucket.get(key), accountId, billingAccessRoleName, billingAccessExternalId, billingS3BucketPrefix, billingS3BucketRegion));
}
}

Expand Down Expand Up @@ -172,7 +173,7 @@ protected void poll() throws Exception {

File file = new File(config.localDir, fileKey.substring(billingFile.prefix.length()));
logger.info("trying to download " + fileKey + "...");
boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.prefix, file, lastProcessed,
boolean downloaded = AwsUtils.downloadFileIfChangedSince(objectSummary.getBucketName(), billingFile.region, billingFile.prefix, file, lastProcessed,
billingFile.accountId, billingFile.accessRoleName, billingFile.externalId);
if (downloaded)
logger.info("downloaded " + fileKey);
Expand All @@ -197,8 +198,8 @@ protected void poll() throws Exception {
logger.info("processing " + monitorFileKey + "...");
File monitorFile = new File(config.localDir, monitorFileKey.substring(monitorFileKey.lastIndexOf("/") + 1));
logger.info("trying to download " + monitorFileKey + "...");
boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.prefix, monitorFile, lastProcessed,
monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
boolean downloaded = AwsUtils.downloadFileIfChangedSince(monitorObjectSummary.getBucketName(), monitorBillingFile.region, monitorBillingFile.prefix,
monitorFile, lastProcessed, monitorBillingFile.accountId, monitorBillingFile.accessRoleName, monitorBillingFile.externalId);
if (downloaded)
logger.info("downloaded " + monitorFile);
else
Expand Down Expand Up @@ -741,13 +742,15 @@ endMilli < lastAlertMillis() + AwsUtils.hourMillis * 24)

private class BillingFile {
final S3ObjectSummary s3ObjectSummary;
final String region;
final String accountId;
final String accessRoleName;
final String externalId;
final String prefix;

BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix) {
BillingFile(S3ObjectSummary s3ObjectSummary, String accountId, String accessRoleName, String externalId, String prefix, String region) {
this.s3ObjectSummary = s3ObjectSummary;
this.region = region;
this.accountId = accountId;
this.accessRoleName = accessRoleName;
this.externalId = externalId;
Expand Down
2 changes: 2 additions & 0 deletions src/java/com/netflix/ice/processor/ProcessorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class ProcessorConfig extends Config {

public final String[] billingAccountIds;
public final String[] billingS3BucketNames;
public final String[] billingS3BucketRegions;
public final String[] billingS3BucketPrefixes;
public final String[] billingAccessRoleNames;
public final String[] billingAccessExternalIds;
Expand Down Expand Up @@ -80,6 +81,7 @@ public ProcessorConfig(
this.costPerMonitorMetricPerHour = 0;

billingS3BucketNames = properties.getProperty(IceOptions.BILLING_S3_BUCKET_NAME).split(",");
billingS3BucketRegions = properties.getProperty(IceOptions.BILLING_S3_BUCKET_REGION).split(",");
billingS3BucketPrefixes = properties.getProperty(IceOptions.BILLING_S3_BUCKET_PREFIX, "").split(",");
billingAccountIds = properties.getProperty(IceOptions.BILLING_PAYER_ACCOUNT_ID, "").split(",");
billingAccessRoleNames = properties.getProperty(IceOptions.BILLING_ACCESS_ROLENAME, "").split(",");
Expand Down
2 changes: 2 additions & 0 deletions src/java/sample.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ ice.ondemandCostAlertEmails=
# modify the following 5 properties according to your billing files configuration. if you have multiple payer accounts, you will need to specify multiple values for each property.
# s3 bucket name where the billing files are. multiple bucket names are delimited by ",". Ice must have read access to billing s3 bucket.
ice.billing_s3bucketname=billing_s3bucketname1,billing_s3bucketname2
# location for the billing bucket. It should be specified for buckets using v4 validation
ice.billing_s3bucketregion=eu-west-1,eu-central-1
# prefix of the billing files. multiple prefixes are delimited by ","
ice.billing_s3bucketprefix=,
# specify your payer account id here if across-accounts IAM role access is used. multiple account ids are delimited by ",". "ice.billing_payerAccountId=,222222222222" means assumed role access is only used for the second bucket.
Expand Down