Skip to content

Commit

Permalink
Merge pull request #309 from asserts/radha/sc-16154
Browse files Browse the repository at this point in the history 
Emit ECS Meta metrics even when no prometheus instrumentation
  • Loading branch information
@jradhakrishnan
jradhakrishnan authored Jul 17, 2023
2 parents be0cde7 + a0921b2 commit 080da4d
Show file tree
Hide file tree
Showing 5 changed files with 122 additions and 92 deletions.
15 changes: 0 additions & 15 deletions config/src/main/java/ai/asserts/aws/config/ScrapeConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,18 +72,6 @@ public class ScrapeConfig {
@Builder.Default
private Map<String, SubnetDetails> primaryExporterByAccount = new TreeMap<>();

@Builder.Default
private Integer listMetricsResultCacheTTLMinutes = 10;

@Builder.Default
private Integer listFunctionsResultCacheTTLMinutes = 5;

@Builder.Default
private Integer getResourcesResultCacheTTLMinutes = 5;

@Builder.Default
private Integer numTaskThreads = 5;

@Builder.Default
private AuthConfig authConfig = new AuthConfig();

Expand All @@ -93,9 +81,6 @@ public class ScrapeConfig {
@Builder.Default
private Integer delay = 0;

@Builder.Default
private Integer logScrapeDelaySeconds = 15;

private TagExportConfig tagExportConfig;

private String alertForwardUrl;
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/ai/asserts/aws/exporter/ECSServiceDiscoveryExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ void writeFile(ScrapeConfig scrapeConfig, List<StaticConfig> targets, String fil

@VisibleForTesting
boolean shouldScrapeTargets(ScrapeConfig scrapeConfig, StaticConfig config) {
if( config.getTargets().isEmpty() ) {
return false;
}
String targetVpc = config.getLabels().getVpcId();
String targetSubnet = config.getLabels().getSubnetId();
boolean vpcOK = scrapeConfig.isDiscoverECSTasksAcrossVPCs() ||
Expand Down
145 changes: 78 additions & 67 deletions src/main/java/ai/asserts/aws/exporter/ECSTaskUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
*/
package ai.asserts.aws.exporter;

import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.AWSApiCallRateLimiter;
import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.TagUtil;
import ai.asserts.aws.TaskExecutorUtil;
import ai.asserts.aws.account.AWSAccount;
Expand Down Expand Up @@ -75,7 +75,8 @@ public class ECSTaskUtil {
public static final String PROMETHEUS_METRIC_PATH_DOCKER_LABEL = "PROMETHEUS_EXPORTER_PATH";


public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceMapper, AWSApiCallRateLimiter rateLimiter,
public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceMapper,
AWSApiCallRateLimiter rateLimiter,
TagUtil tagUtil, TaskExecutorUtil taskExecutorUtil) {
this.awsClientProvider = awsClientProvider;
this.resourceMapper = resourceMapper;
Expand All @@ -86,17 +87,6 @@ public ECSTaskUtil(AWSClientProvider awsClientProvider, ResourceMapper resourceM
envName = getInstallEnvName();
}

@VisibleForTesting
String getInstallEnvName() {
final String envName;
if (System.getenv(INSTALLED_ENV_NAME) != null) {
envName = System.getenv(INSTALLED_ENV_NAME);
} else {
envName = null;
}
return envName;
}

public boolean hasAllInfo(Task task) {
return "RUNNING".equals(task.lastStatus()) && task.hasAttachments() && task.attachments()
.stream()
Expand Down Expand Up @@ -142,33 +132,26 @@ public List<StaticConfig> buildScrapeTargets(AWSAccount account, ScrapeConfig sc
Optional<String> portFromLabel = getDockerLabel(cD, PROMETHEUS_PORT_DOCKER_LABEL);
labelsBuilder.availabilityZone(task.availabilityZone());
String jobName = cD.name();
labelsBuilder.container(jobName);
labelsBuilder.job(jobName);
if (pathFromLabel.isPresent() && portFromLabel.isPresent()) {
log.debug("Found prometheus port={}, path={} from docker labels for container {}/{}",
portFromLabel.get(),
pathFromLabel.get(),
taskDefinition.taskDefinitionArn(),
cD.name());
Labels labels = labelsBuilder
.job(jobName)
.metricsPath(pathFromLabel.get())
.container(cD.name())
.build();

labels.populateMapEntries();
labels.putAll(tagLabels);
StaticConfig staticConfig = targetsByLabel.computeIfAbsent(
labels, k -> StaticConfig.builder().labels(labels).build());
StaticConfig staticConfig = buildStaticConfig(tagLabels, targetsByLabel, cD, labels);
staticConfig.getTargets().add(format("%s:%s", ipAddress, portFromLabel.get()));
if (cD.logConfiguration() != null) {
LogConfig logConfig = LogConfig.builder()
.logDriver(cD.logConfiguration().logDriver().toString())
.options(cD.logConfiguration().options()).build();
staticConfig.getLogConfigs().add(logConfig);
}
} else {
log.warn("Docker labels for prometheus port and path not found for container {}/{}",
taskDefinition.taskDefinitionArn(),
cD.name());
Labels labels = labelsBuilder.build();
buildStaticConfig(tagLabels, targetsByLabel, cD, labels);
}
});
} else {
Expand All @@ -187,6 +170,58 @@ public List<StaticConfig> buildScrapeTargets(AWSAccount account, ScrapeConfig sc
return targets;
}

public SubnetDetails getSubnetDetails(Resource taskResource) {
EcsClient ecsClient = awsClientProvider.getECSClient(taskResource.getRegion(), AWSAccount.builder()
.accountId(taskResource.getAccount())
.build());
DescribeTasksResponse response = rateLimiter.doWithRateLimit("EcsClient/describeTasks",
ImmutableSortedMap.of(
SCRAPE_ACCOUNT_ID_LABEL, taskResource.getAccount(),
SCRAPE_REGION_LABEL, taskResource.getRegion(),
SCRAPE_OPERATION_LABEL, "EcsClient/describeTasks"),
() -> ecsClient.describeTasks(DescribeTasksRequest.builder()
.cluster(taskResource.getChildOf().getName())
.tasks(taskResource.getArn())
.build()));
if (response.hasTasks()) {
return response.tasks().get(0).attachments().stream()
.filter(attachment -> attachment.type().equals("ElasticNetworkInterface"))
.findFirst()
.flatMap(attachment -> attachment.details().stream()
.filter(kv -> kv.name().equals("subnetId")).findFirst())
.map(kv -> {
AtomicReference<String> vpcId = new AtomicReference<>("");
AtomicReference<String> subnetId = new AtomicReference<>("");
subnetId.set(kv.value());
vpcId.set(subnetIdMap.computeIfAbsent(subnetId.get(), kk ->
getVpcId(taskResource, subnetId)));
return SubnetDetails.builder()
.vpcId(vpcId.get())
.subnetId(subnetId.get())
.build();
}).orElse(null);
}
log.warn("Failed to find description for {}", taskResource);
return null;
}

@VisibleForTesting
String getEnv(AWSAccount account) {
return account.getName() != null ? account.getName() : envName != null ? envName : account.getAccountId();
}

@VisibleForTesting
String getInstallEnvName() {
final String envName;
if (System.getenv(INSTALLED_ENV_NAME) != null) {
envName = System.getenv(INSTALLED_ENV_NAME);
} else {
envName = null;
}
return envName;
}


@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Optional<String> service, Task task) {
Resource taskDefResource = resourceMapper.map(task.taskDefinitionArn())
Expand Down Expand Up @@ -232,8 +267,19 @@ private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Opt
return labelsBuilder;
}

private String getEnv(AWSAccount account) {
return envName != null ? envName : account.getName() != null ? account.getName() : account.getAccountId();
private StaticConfig buildStaticConfig(Map<String, String> tagLabels, Map<Labels, StaticConfig> targetsByLabel,
ContainerDefinition cD, Labels labels) {
labels.populateMapEntries();
labels.putAll(tagLabels);
StaticConfig staticConfig = targetsByLabel.computeIfAbsent(
labels, k -> StaticConfig.builder().labels(labels).build());
if (cD.logConfiguration() != null) {
LogConfig logConfig = LogConfig.builder()
.logDriver(cD.logConfiguration().logDriver().toString())
.options(cD.logConfiguration().options()).build();
staticConfig.getLogConfigs().add(logConfig);
}
return staticConfig;
}

private String getIPAddress(Task task) {
Expand All @@ -249,39 +295,11 @@ private String getIPAddress(Task task) {
return ipAddress;
}

public SubnetDetails getSubnetDetails(Resource taskResource) {
EcsClient ecsClient = awsClientProvider.getECSClient(taskResource.getRegion(), AWSAccount.builder()
.accountId(taskResource.getAccount())
.build());
DescribeTasksResponse response = rateLimiter.doWithRateLimit("EcsClient/describeTasks",
ImmutableSortedMap.of(
SCRAPE_ACCOUNT_ID_LABEL, taskResource.getAccount(),
SCRAPE_REGION_LABEL, taskResource.getRegion(),
SCRAPE_OPERATION_LABEL, "EcsClient/describeTasks"),
() -> ecsClient.describeTasks(DescribeTasksRequest.builder()
.cluster(taskResource.getChildOf().getName())
.tasks(taskResource.getArn())
.build()));
if (response.hasTasks()) {
return response.tasks().get(0).attachments().stream()
.filter(attachment -> attachment.type().equals("ElasticNetworkInterface"))
.findFirst()
.flatMap(attachment -> attachment.details().stream()
.filter(kv -> kv.name().equals("subnetId")).findFirst())
.map(kv -> {
AtomicReference<String> vpcId = new AtomicReference<>("");
AtomicReference<String> subnetId = new AtomicReference<>("");
subnetId.set(kv.value());
vpcId.set(subnetIdMap.computeIfAbsent(subnetId.get(), kk ->
getVpcId(taskResource, subnetId)));
return SubnetDetails.builder()
.vpcId(vpcId.get())
.subnetId(subnetId.get())
.build();
}).orElse(null);
}
log.warn("Failed to find description for {}", taskResource);
return null;
private Optional<String> getDockerLabel(ContainerDefinition container, String labelName) {
return container.dockerLabels().entrySet().stream()
.filter(entry -> entry.getKey().equals(labelName))
.map(Map.Entry::getValue)
.findFirst();
}

private SubnetDetails getSubnetDetails(Task task, Resource taskResource) {
Expand Down Expand Up @@ -321,11 +339,4 @@ private String getVpcId(Resource taskResource, AtomicReference<String> subnetId)
r.subnets().stream().findFirst().ifPresent(subnet -> id.set(subnet.vpcId()));
return id.get();
}

Optional<String> getDockerLabel(ContainerDefinition container, String labelName) {
return container.dockerLabels().entrySet().stream()
.filter(entry -> entry.getKey().equals(labelName))
.map(Map.Entry::getValue)
.findFirst();
}
}
31 changes: 23 additions & 8 deletions src/test/java/ai/asserts/aws/exporter/ECSServiceDiscoveryExporterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.easymock.EasyMockSupport;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -238,36 +239,49 @@ void identifySubnetsToScrape() {
.vpcId("vpc-1")
.subnetId("subnet-1")
.build());
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
StaticConfig staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-1")
.build())
.build()));
.build();

// Without target
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// With target
staticConfig.getTargets().add("1.2.3.4:8080");
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Same VPC, different subnet. But subnet configured to be scraped
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-2")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertTrue(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Same VPC, different subnet. But subnet not configured to be scraped
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-1")
.subnetId("subnet-3")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));

// Different VPC
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, StaticConfig.builder()
staticConfig = StaticConfig.builder()
.labels(Labels.builder()
.vpcId("vpc-2")
.subnetId("subnet-1")
.build())
.build()));
.build();
staticConfig.getTargets().add("1.2.3.4:8080");
assertFalse(testClass.shouldScrapeTargets(scrapeConfig, staticConfig));
verifyAll();
}

Expand All @@ -283,6 +297,7 @@ public void run() throws Exception {
expect(ecsTaskProvider.getScrapeTargets()).andReturn(ImmutableList.of(mockStaticConfig, mockStaticConfig));

expect(scrapeConfig.isLogECSTargets()).andReturn(true);
expect(mockStaticConfig.getTargets()).andReturn(ImmutableSet.of("1.2.3.4:8080")).anyTimes();
expect(mockStaticConfig.getLabels()).andReturn(mockLabels).anyTimes();
expect(mockLabels.getVpcId()).andReturn("vpc-id").anyTimes();
expect(mockLabels.getSubnetId()).andReturn("subnet-id").anyTimes();
Expand Down
20 changes: 18 additions & 2 deletions src/test/java/ai/asserts/aws/exporter/ECSTaskUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
*/
package ai.asserts.aws.exporter;

import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.AWSApiCallRateLimiter;
import ai.asserts.aws.AWSClientProvider;
import ai.asserts.aws.TagUtil;
import ai.asserts.aws.TaskExecutorUtil;
import ai.asserts.aws.TestTaskThreadPool;
Expand Down Expand Up @@ -66,6 +66,7 @@ public class ECSTaskUtilTest extends EasyMockSupport {
private TagUtil tagUtil;
private ScrapeConfig scrapeConfig;
private AWSAccount account;
private String defaultEnvName;

@BeforeEach
public void setup() {
Expand All @@ -85,9 +86,15 @@ public void setup() {
Ec2Client ec2Client = mock(Ec2Client.class);


defaultEnvName = "dev";
testClass = new ECSTaskUtil(awsClientProvider, resourceMapper,
rateLimiter, tagUtil,
taskExecutorUtil);
taskExecutorUtil) {
@Override
String getInstallEnvName() {
return defaultEnvName;
}
};

expect(awsClientProvider.getEc2Client(anyString(), anyObject())).andReturn(ec2Client).anyTimes();
expect(ec2Client.describeSubnets(DescribeSubnetsRequest.builder()
Expand Down Expand Up @@ -144,6 +151,15 @@ public void hasAllInfo_true() {
verifyAll();
}

@Test
public void getEnvName() {
assertEquals("prod", testClass.getEnv(AWSAccount.builder()
.name("prod")
.build()));
assertEquals(defaultEnvName, testClass.getEnv(AWSAccount.builder()
.build()));
}

@Test
public void containerWithDockerLabels() {
expect(resourceMapper.map("task-def-arn")).andReturn(Optional.of(taskDef));
Expand Down

0 comments on commit 080da4d

Please sign in to comment.