Aws monitoring and SNS + SQS -> Lambda + Global edge type constraints (…

…#975)
klothoplatform · Mar 25, 2024 · 0145087 · 0145087
1 parent bf8a303
commit 0145087
Show file tree

Hide file tree

Showing 43 changed files with 1,266 additions and 39 deletions.
diff --git a/pkg/engine/constraints.go b/pkg/engine/constraints.go
@@ -138,6 +138,15 @@ func applyEdgeConstraint(ctx solution_context.SolutionContext, constraint constr
 		}
 	}
 
+	if constraint.Target.Source.Name == "" || constraint.Target.Target.Name == "" {
+		if constraint.Target.Source.Name == "" && constraint.Target.Target.Name == "" {
+			return fmt.Errorf("source and target names are empty")
+		}
+		// This is considered a global constraint for the type which does not have a name and
+		// will be applied anytime a new resource is added to the graph
+		return nil
+	}
+
 	switch constraint.Operator {
 	case constraints.AddConstraintOperator:
 		return ctx.OperationalView().AddEdge(constraint.Target.Source, constraint.Target.Target)

diff --git a/pkg/engine/operational_eval/graph.go b/pkg/engine/operational_eval/graph.go
@@ -287,6 +287,11 @@ func (eval *Evaluator) RemoveResource(id construct.ResourceId) error {
 
 		case *graphStateVertex:
 			checkStates.Add(v.Key())
+
+		case *resourceRuleVertex:
+			if v.Resource == id {
+				errs = errors.Join(errs, eval.removeKey(v.Key()))
+			}
 		}
 	}
 	if errs != nil {

diff --git a/pkg/engine/testdata/delete_resource_and_iacdeps.expect.yaml b/pkg/engine/testdata/delete_resource_and_iacdeps.expect.yaml
@@ -7,6 +7,9 @@ resources:
             GLOBAL_KLOTHO_TAG: ""
             RESOURCE_NAME: api_stage-0
     aws:ecs_cluster:ecs_cluster-0:
+        ClusterSettings:
+            - Name: containerInsights
+              Value: enabled
         Tags:
             GLOBAL_KLOTHO_TAG: ""
             RESOURCE_NAME: ecs_cluster-0

diff --git a/pkg/engine/testdata/ecs_rds.deployment-policy.json b/pkg/engine/testdata/ecs_rds.deployment-policy.json
@@ -18,6 +18,15 @@
             "Effect": "Allow",
             "Resource": "*"
         },
+        {
+            "Action": [
+                "dynamodb:CreateTable",
+                "dynamodb:DeleteTable",
+                "dynamodb:UpdateTable"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        },
         {
             "Action": [
                 "ecs:CreateCluster",

diff --git a/pkg/engine/testdata/ecs_rds.expect.yaml b/pkg/engine/testdata/ecs_rds.expect.yaml
@@ -33,7 +33,58 @@ resources:
             GLOBAL_KLOTHO_TAG: ""
             RESOURCE_NAME: ecs_service_0
         TaskDefinition: aws:ecs_task_definition:ecs_service_0
+    aws:cloudwatch_alarm:ecs_service_0-CPUUtilization:
+        ActionsEnabled: true
+        AlarmDescription: This metric checks for CPUUtilization in the ECS service
+        ComparisonOperator: GreaterThanOrEqualToThreshold
+        Dimensions:
+            ClusterName: aws:ecs_cluster:ecs_cluster-0#Id
+            ServiceName: aws:ecs_service:ecs_service_0#Name
+        EvaluationPeriods: 2
+        MetricName: CPUUtilization
+        Namespace: AWS/ECS
+        Period: 60
+        Statistic: Average
+        Tags:
+            GLOBAL_KLOTHO_TAG: ""
+            RESOURCE_NAME: ecs_service_0-CPUUtilization
+        Threshold: 90
+    aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization:
+        ActionsEnabled: true
+        AlarmDescription: This metric checks for MemoryUtilization in the ECS service
+        ComparisonOperator: GreaterThanOrEqualToThreshold
+        Dimensions:
+            ClusterName: aws:ecs_cluster:ecs_cluster-0#Id
+            ServiceName: aws:ecs_service:ecs_service_0#Name
+        EvaluationPeriods: 2
+        MetricName: MemoryUtilization
+        Namespace: AWS/ECS
+        Period: 60
+        Statistic: Average
+        Tags:
+            GLOBAL_KLOTHO_TAG: ""
+            RESOURCE_NAME: ecs_service_0-MemoryUtilization
+        Threshold: 90
+    aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount:
+        ActionsEnabled: true
+        AlarmDescription: This metric checks for any stopped tasks in the ECS service
+        ComparisonOperator: LessThanThreshold
+        Dimensions:
+            ClusterName: aws:ecs_cluster:ecs_cluster-0#Id
+            ServiceName: aws:ecs_service:ecs_service_0#Name
+        EvaluationPeriods: 1
+        MetricName: RunningTaskCount
+        Namespace: AWS/ECS
+        Period: 60
+        Statistic: Average
+        Tags:
+            GLOBAL_KLOTHO_TAG: ""
+            RESOURCE_NAME: ecs_service_0-RunningTaskCount
+        Threshold: 1
     aws:ecs_cluster:ecs_cluster-0:
+        ClusterSettings:
+            - Name: containerInsights
+              Value: enabled
         Tags:
             GLOBAL_KLOTHO_TAG: ""
             RESOURCE_NAME: ecs_cluster-0
@@ -71,6 +122,51 @@ resources:
             GLOBAL_KLOTHO_TAG: ""
             RESOURCE_NAME: ecs_service_0
         TaskRole: aws:iam_role:ecs_service_0-execution-role
+    aws:cloudwatch_dashboard:cloudwatch_dashboard-0:
+        DashboardBody:
+            Widgets:
+                - Height: 6
+                  Properties:
+                    Annotations:
+                        Alarms:
+                            - aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount#Arn
+                    Region: aws:region:region-0#Name
+                  Type: metric
+                  Width: 6
+                - Height: 6
+                  Properties:
+                    Alarms:
+                        - aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount#Arn
+                  Type: alarm
+                  Width: 6
+                - Height: 6
+                  Properties:
+                    Annotations:
+                        Alarms:
+                            - aws:cloudwatch_alarm:ecs_service_0-CPUUtilization#Arn
+                    Region: aws:region:region-0#Name
+                  Type: metric
+                  Width: 6
+                - Height: 6
+                  Properties:
+                    Alarms:
+                        - aws:cloudwatch_alarm:ecs_service_0-CPUUtilization#Arn
+                  Type: alarm
+                  Width: 6
+                - Height: 6
+                  Properties:
+                    Annotations:
+                        Alarms:
+                            - aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization#Arn
+                    Region: aws:region:region-0#Name
+                  Type: metric
+                  Width: 6
+                - Height: 6
+                  Properties:
+                    Alarms:
+                        - aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization#Arn
+                  Type: alarm
+                  Width: 6
     aws:ecr_image:ecs_service_0-ecs_service_0:
         Context: .
         Dockerfile: ecs_service_0-ecs_service_0.Dockerfile
@@ -289,10 +385,19 @@ resources:
 edges:
     aws:security_group:vpc-0:ecs_service_0-security_group -> aws:ecs_service:ecs_service_0:
     aws:security_group:vpc-0:ecs_service_0-security_group -> aws:vpc:vpc-0:
+    aws:ecs_service:ecs_service_0 -> aws:cloudwatch_alarm:ecs_service_0-CPUUtilization:
+    aws:ecs_service:ecs_service_0 -> aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization:
+    aws:ecs_service:ecs_service_0 -> aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount:
     aws:ecs_service:ecs_service_0 -> aws:ecs_cluster:ecs_cluster-0:
     aws:ecs_service:ecs_service_0 -> aws:ecs_task_definition:ecs_service_0:
     aws:ecs_service:ecs_service_0 -> aws:subnet:vpc-0:subnet-0:
     aws:ecs_service:ecs_service_0 -> aws:subnet:vpc-0:subnet-1:
+    aws:cloudwatch_alarm:ecs_service_0-CPUUtilization -> aws:cloudwatch_dashboard:cloudwatch_dashboard-0:
+    aws:cloudwatch_alarm:ecs_service_0-CPUUtilization -> aws:region:region-0:
+    aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization -> aws:cloudwatch_dashboard:cloudwatch_dashboard-0:
+    aws:cloudwatch_alarm:ecs_service_0-MemoryUtilization -> aws:region:region-0:
+    aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount -> aws:cloudwatch_dashboard:cloudwatch_dashboard-0:
+    aws:cloudwatch_alarm:ecs_service_0-RunningTaskCount -> aws:region:region-0:
     aws:ecs_task_definition:ecs_service_0 -> aws:ecr_image:ecs_service_0-ecs_service_0:
     aws:ecs_task_definition:ecs_service_0 -> aws:iam_role:ecs_service_0-execution-role:
     aws:ecs_task_definition:ecs_service_0 -> aws:log_group:ecs_service_0-log-group:

diff --git a/pkg/engine/testdata/ecs_rds.iac-viz.yaml b/pkg/engine/testdata/ecs_rds.iac-viz.yaml
@@ -1,12 +1,11 @@
 provider: aws
 resources:
-  ecs_service/ecs_service_0:
+  cloudwatch_dashboard/cloudwatch_dashboard-0:
 
-  ecs_service/ecs_service_0 -> ecs_cluster/ecs_cluster-0:
-  ecs_service/ecs_service_0 -> ecs_task_definition/ecs_service_0:
-  ecs_service/ecs_service_0 -> aws:security_group:vpc-0/ecs_service_0-security_group:
-  ecs_service/ecs_service_0 -> aws:subnet:vpc-0/subnet-0:
-  ecs_service/ecs_service_0 -> aws:subnet:vpc-0/subnet-1:
+  cloudwatch_dashboard/cloudwatch_dashboard-0 -> cloudwatch_alarm/ecs_service_0-cpuutilization:
+  cloudwatch_dashboard/cloudwatch_dashboard-0 -> cloudwatch_alarm/ecs_service_0-memoryutilization:
+  cloudwatch_dashboard/cloudwatch_dashboard-0 -> cloudwatch_alarm/ecs_service_0-runningtaskcount:
+  cloudwatch_dashboard/cloudwatch_dashboard-0 -> region/region-0:
   route_table_association/subnet-0-subnet-0-route_table:
 
   route_table_association/subnet-0-subnet-0-route_table -> aws:route_table:vpc-0/subnet-0-route_table:
@@ -23,18 +22,18 @@ resources:
 
   route_table_association/subnet-3-subnet-3-route_table -> aws:route_table:vpc-0/subnet-3-route_table:
   route_table_association/subnet-3-subnet-3-route_table -> aws:subnet:vpc-0/subnet-3:
-  ecs_cluster/ecs_cluster-0:
+  cloudwatch_alarm/ecs_service_0-cpuutilization:
 
-  ecs_task_definition/ecs_service_0:
+  cloudwatch_alarm/ecs_service_0-cpuutilization -> ecs_service/ecs_service_0:
+  cloudwatch_alarm/ecs_service_0-cpuutilization -> region/region-0:
+  cloudwatch_alarm/ecs_service_0-memoryutilization:
 
-  ecs_task_definition/ecs_service_0 -> ecr_image/ecs_service_0-ecs_service_0:
-  ecs_task_definition/ecs_service_0 -> iam_role/ecs_service_0-execution-role:
-  ecs_task_definition/ecs_service_0 -> log_group/ecs_service_0-log-group:
-  ecs_task_definition/ecs_service_0 -> rds_instance/rds-instance-2:
-  ecs_task_definition/ecs_service_0 -> region/region-0:
-  aws:security_group:vpc-0/ecs_service_0-security_group:
+  cloudwatch_alarm/ecs_service_0-memoryutilization -> ecs_service/ecs_service_0:
+  cloudwatch_alarm/ecs_service_0-memoryutilization -> region/region-0:
+  cloudwatch_alarm/ecs_service_0-runningtaskcount:
 
-  aws:security_group:vpc-0/ecs_service_0-security_group -> vpc/vpc-0:
+  cloudwatch_alarm/ecs_service_0-runningtaskcount -> ecs_service/ecs_service_0:
+  cloudwatch_alarm/ecs_service_0-runningtaskcount -> region/region-0:
   aws:route_table:vpc-0/subnet-0-route_table:
 
   aws:route_table:vpc-0/subnet-0-route_table -> aws:nat_gateway:subnet-2/subnet-0-route_table-nat_gateway:
@@ -51,14 +50,13 @@ resources:
 
   aws:route_table:vpc-0/subnet-3-route_table -> aws:internet_gateway:vpc-0/internet_gateway-0:
   aws:route_table:vpc-0/subnet-3-route_table -> vpc/vpc-0:
-  ecr_image/ecs_service_0-ecs_service_0:
-
-  ecr_image/ecs_service_0-ecs_service_0 -> ecr_repo/ecs_service_0-ecs_service_0-ecr_repo:
-  iam_role/ecs_service_0-execution-role:
-
-  iam_role/ecs_service_0-execution-role -> rds_instance/rds-instance-2:
-  log_group/ecs_service_0-log-group:
+  ecs_service/ecs_service_0:
 
+  ecs_service/ecs_service_0 -> ecs_cluster/ecs_cluster-0:
+  ecs_service/ecs_service_0 -> ecs_task_definition/ecs_service_0:
+  ecs_service/ecs_service_0 -> aws:security_group:vpc-0/ecs_service_0-security_group:
+  ecs_service/ecs_service_0 -> aws:subnet:vpc-0/subnet-0:
+  ecs_service/ecs_service_0 -> aws:subnet:vpc-0/subnet-1:
   aws:nat_gateway:subnet-2/subnet-0-route_table-nat_gateway:
 
   aws:nat_gateway:subnet-2/subnet-0-route_table-nat_gateway -> elastic_ip/subnet-0-route_table-nat_gateway-elastic_ip:
@@ -70,12 +68,18 @@ resources:
   aws:internet_gateway:vpc-0/internet_gateway-0:
 
   aws:internet_gateway:vpc-0/internet_gateway-0 -> vpc/vpc-0:
-  ecr_repo/ecs_service_0-ecs_service_0-ecr_repo:
+  ecs_cluster/ecs_cluster-0:
 
-  rds_instance/rds-instance-2:
+  ecs_task_definition/ecs_service_0:
 
-  rds_instance/rds-instance-2 -> rds_subnet_group/rds_subnet_group-0:
-  rds_instance/rds-instance-2 -> aws:security_group:vpc-0/rds-instance-2-security_group:
+  ecs_task_definition/ecs_service_0 -> ecr_image/ecs_service_0-ecs_service_0:
+  ecs_task_definition/ecs_service_0 -> iam_role/ecs_service_0-execution-role:
+  ecs_task_definition/ecs_service_0 -> log_group/ecs_service_0-log-group:
+  ecs_task_definition/ecs_service_0 -> rds_instance/rds-instance-2:
+  ecs_task_definition/ecs_service_0 -> region/region-0:
+  aws:security_group:vpc-0/ecs_service_0-security_group:
+
+  aws:security_group:vpc-0/ecs_service_0-security_group -> vpc/vpc-0:
   elastic_ip/subnet-0-route_table-nat_gateway-elastic_ip:
 
   aws:subnet:vpc-0/subnet-2:
@@ -88,6 +92,20 @@ resources:
 
   aws:subnet:vpc-0/subnet-3 -> aws:availability_zone:region-0/availability_zone-1:
   aws:subnet:vpc-0/subnet-3 -> vpc/vpc-0:
+  ecr_image/ecs_service_0-ecs_service_0:
+
+  ecr_image/ecs_service_0-ecs_service_0 -> ecr_repo/ecs_service_0-ecs_service_0-ecr_repo:
+  iam_role/ecs_service_0-execution-role:
+
+  iam_role/ecs_service_0-execution-role -> rds_instance/rds-instance-2:
+  log_group/ecs_service_0-log-group:
+
+  ecr_repo/ecs_service_0-ecs_service_0-ecr_repo:
+
+  rds_instance/rds-instance-2:
+
+  rds_instance/rds-instance-2 -> rds_subnet_group/rds_subnet_group-0:
+  rds_instance/rds-instance-2 -> aws:security_group:vpc-0/rds-instance-2-security_group:
   rds_subnet_group/rds_subnet_group-0:
 
   rds_subnet_group/rds_subnet_group-0 -> aws:subnet:vpc-0/subnet-0:

diff --git a/pkg/engine/testdata/idempotent_constraints.deployment-policy.json b/pkg/engine/testdata/idempotent_constraints.deployment-policy.json
@@ -18,6 +18,15 @@
             "Effect": "Allow",
             "Resource": "*"
         },
+        {
+            "Action": [
+                "dynamodb:CreateTable",
+                "dynamodb:DeleteTable",
+                "dynamodb:UpdateTable"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        },
         {
             "Action": [
                 "ecs:CreateCluster",