diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1b440d9ea..061512721 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,6 +10,8 @@ repos: - id: double-quote-string-fixer - id: trailing-whitespace - id: end-of-file-fixer + exclude: ^(cdk)/tests/unit/snapshots - id: pretty-format-json args: ['--autofix', '--no-sort-keys', '--indent=4', '--no-ensure-ascii'] + exclude: ^(cdk)/tests/unit/snapshots - id: check-yaml diff --git a/cdk/infrastructure/config.py b/cdk/infrastructure/config.py index ca2ae47f1..60077fb84 100644 --- a/cdk/infrastructure/config.py +++ b/cdk/infrastructure/config.py @@ -240,8 +240,8 @@ 'max_scaling_capacity': 2, }, 'indexing_service': { - 'cpu': 512, - 'memory_limit_mib': 1024, + 'cpu': 256, + 'memory_limit_mib': 512, 'min_scaling_capacity': 1, 'max_scaling_capacity': 2, }, @@ -305,8 +305,8 @@ 'max_scaling_capacity': 2, }, 'indexing_service': { - 'cpu': 512, - 'memory_limit_mib': 1024, + 'cpu': 256, + 'memory_limit_mib': 512, 'min_scaling_capacity': 1, 'max_scaling_capacity': 2, }, @@ -369,8 +369,8 @@ 'max_scaling_capacity': 2, }, 'indexing_service': { - 'cpu': 512, - 'memory_limit_mib': 1024, + 'cpu': 256, + 'memory_limit_mib': 512, 'min_scaling_capacity': 1, 'max_scaling_capacity': 2, }, diff --git a/cdk/infrastructure/constructs/indexer.py b/cdk/infrastructure/constructs/indexer.py index 9995214f0..750e08d99 100644 --- a/cdk/infrastructure/constructs/indexer.py +++ b/cdk/infrastructure/constructs/indexer.py @@ -125,9 +125,11 @@ def __init__( self._define_opensearch() self._define_docker_assets() self._define_invalidation_service() + self._remove_cpu_scaling_from_invalidation_service() self._allow_invalidation_service_to_write_to_invalidation_queue() self._define_portal_credentials() self._define_indexing_service() + self._remove_cpu_scaling_from_indexing_service() self._allow_invalidation_service_to_connect_to_opensearch() self._allow_indexing_service_to_connect_to_opensearch() self._add_alarms_to_invalidation_service() @@ -177,6 +179,15 @@ def _define_invalidation_service(self) -> None: ), ) + def _remove_cpu_scaling_from_invalidation_service(self) -> None: + self.invalidation_service.service.node.find_child( + 'TaskCount' + ).node.find_child( + 'Target' + ).node.try_remove_child( + 'CpuScaling' + ) + def _allow_invalidation_service_to_write_to_invalidation_queue(self) -> None: self.props.invalidation_queue.queue.grant_send_messages( self.invalidation_service.task_definition.task_role @@ -226,6 +237,15 @@ def _define_indexing_service(self) -> None: ), ) + def _remove_cpu_scaling_from_indexing_service(self) -> None: + self.indexing_service.service.node.find_child( + 'TaskCount' + ).node.find_child( + 'Target' + ).node.try_remove_child( + 'CpuScaling' + ) + def _allow_invalidation_service_to_connect_to_opensearch(self) -> None: self.invalidation_service.service.connections.allow_to( self.opensearch.domain, diff --git a/cdk/requirements-dev.txt b/cdk/requirements-dev.txt index f94700272..6d1719ba3 100644 --- a/cdk/requirements-dev.txt +++ b/cdk/requirements-dev.txt @@ -3,3 +3,4 @@ pytest-instafail==0.4.2 pytest-mock==2.0.0 pytest-cov==2.8.1 mypy==0.950 +pytest-snapshot==0.9.0 diff --git a/cdk/tests/unit/snapshots/test_constructs_indexer/test_constructs_indexer_indexer_match_snapshot/indexer_constructs_template.json b/cdk/tests/unit/snapshots/test_constructs_indexer/test_constructs_indexer_indexer_match_snapshot/indexer_constructs_template.json new file mode 100644 index 000000000..d56080701 --- /dev/null +++ b/cdk/tests/unit/snapshots/test_constructs_indexer/test_constructs_indexer_indexer_match_snapshot/indexer_constructs_template.json @@ -0,0 +1,2879 @@ +{ + "Outputs": { + "ExportsOutputFnGetAttOpensearchDomainCED7C974DomainEndpoint60526A7F": { + "Export": { + "Name": "Default:ExportsOutputFnGetAttOpensearchDomainCED7C974DomainEndpoint60526A7F" + }, + "Value": { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "DomainEndpoint" + ] + } + }, + "ExportsOutputFnGetAttOpensearchDomainSecurityGroup046A436DGroupId70655FA5": { + "Export": { + "Name": "Default:ExportsOutputFnGetAttOpensearchDomainSecurityGroup046A436DGroupId70655FA5" + }, + "Value": { + "Fn::GetAtt": [ + "OpensearchDomainSecurityGroup046A436D", + "GroupId" + ] + } + }, + "IndexerIndexingServiceSQSQueueArnD870A629": { + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "Arn" + ] + } + }, + "IndexerIndexingServiceSQSQueueE5945225": { + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "QueueName" + ] + } + }, + "IndexerInvalidationServiceSQSQueue0379F78F": { + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "QueueName" + ] + } + }, + "IndexerInvalidationServiceSQSQueueArnFAF672D3": { + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "Arn" + ] + } + }, + "TestApplicationLoadBalancedFargateServiceLoadBalancerDNS10E9B084": { + "Value": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceLBC13F6669", + "DNSName" + ] + } + }, + "TestApplicationLoadBalancedFargateServiceServiceURL6371973E": { + "Value": { + "Fn::Join": [ + "", + [ + "http://", + { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceLBC13F6669", + "DNSName" + ] + } + ] + ] + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]", + "Type": "AWS::SSM::Parameter::Value" + } + }, + "Resources": { + "AWS679f53fac002430cb0da5b7982bd22872D164C4C": { + "DependsOn": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "099a3112386fa620b3069790dcf92c8e64bb5341760a8b0a76bfa843f8ad41df.zip" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2", + "Arn" + ] + }, + "Runtime": "nodejs18.x", + "Timeout": 120 + }, + "Type": "AWS::Lambda::Function" + }, + "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Type": "AWS::IAM::Role" + }, + "DownloadManagedPolicyF8118CAF": { + "Properties": { + "Description": "", + "Path": "/", + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": "arn:aws:s3:::some-test-bucket/" + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::ManagedPolicy" + }, + "EcsDefaultClusterMnL3mNNYNTestVpc4872C696": { + "Type": "AWS::ECS::Cluster" + }, + "IndexerIndexingServiceAlarmsCPUAlarmD2583FA3": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClusterName", + "Value": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + } + }, + { + "Name": "ServiceName", + "Value": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingFargateServiceDF01B908", + "Name" + ] + } + } + ], + "EvaluationPeriods": 2, + "MetricName": "CPUUtilization", + "Namespace": "AWS/ECS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Average", + "Threshold": 95 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerIndexingServiceAlarmsMemoryAlarm7DCE3B46": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClusterName", + "Value": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + } + }, + { + "Name": "ServiceName", + "Value": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingFargateServiceDF01B908", + "Name" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "MemoryUtilization", + "Namespace": "AWS/ECS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Average", + "Threshold": 80 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceDF01B908": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "Cluster": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + }, + "DeploymentConfiguration": { + "Alarms": { + "AlarmNames": [], + "Enable": false, + "Rollback": false + }, + "DeploymentCircuitBreaker": { + "Enable": true, + "Rollback": true + }, + "MaximumPercent": 200, + "MinimumHealthyPercent": 50 + }, + "DeploymentController": { + "Type": "ECS" + }, + "EnableECSManagedTags": false, + "EnableExecuteCommand": true, + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingFargateServiceSecurityGroup9D88A792", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "TestVpcpublicSubnet1Subnet4F70BC85" + }, + { + "Ref": "TestVpcpublicSubnet2Subnet96FF72E6" + } + ] + } + }, + "ServiceName": "IndexingService", + "TaskDefinition": { + "Ref": "IndexerIndexingServiceQueueProcessingTaskDef248DB181" + } + }, + "Type": "AWS::ECS::Service" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceSecurityGroup9D88A792": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "GroupDescription": "Default/Indexer/IndexingService/QueueProcessingFargateService/SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetD4009D0B": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "MaxCapacity": 2, + "MinCapacity": 1, + "ResourceId": { + "Fn::Join": [ + "", + [ + "service/", + { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + }, + "/", + { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingFargateServiceDF01B908", + "Name" + ] + } + ] + ] + }, + "RoleARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/aws-service-role/ecs.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_ECSService" + ] + ] + }, + "ScalableDimension": "ecs:service:DesiredCount", + "ServiceNamespace": "ecs" + }, + "Type": "AWS::ApplicationAutoScaling::ScalableTarget" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerAlarm0E5C46A1": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "AlarmActions": [ + { + "Ref": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicy53492B71" + } + ], + "AlarmDescription": "Lower threshold scaling alarm", + "ComparisonOperator": "LessThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 0 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicy53492B71": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "PolicyName": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicyD75DA3EC", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetD4009D0B" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalUpperBound": 0, + "ScalingAdjustment": -1 + } + ] + } + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperAlarm48FD67D4": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "AlarmActions": [ + { + "Ref": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicyA6E16883" + } + ], + "AlarmDescription": "Upper threshold scaling alarm", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicyA6E16883": { + "DependsOn": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + ], + "Properties": { + "PolicyName": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy446651A0", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "IndexerIndexingServiceQueueProcessingFargateServiceTaskCountTargetD4009D0B" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalLowerBound": 0, + "MetricIntervalUpperBound": 999, + "ScalingAdjustment": 1 + }, + { + "MetricIntervalLowerBound": 999, + "ScalingAdjustment": 2 + } + ] + } + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy" + }, + "IndexerIndexingServiceQueueProcessingTaskDef248DB181": { + "Properties": { + "ContainerDefinitions": [ + { + "Command": [ + "run-indexing-service" + ], + "Environment": [ + { + "Name": "INVALIDATION_QUEUE_URL", + "Value": { + "Ref": "InvalidationQueue8614463D" + } + }, + { + "Name": "OPENSEARCH_URL", + "Value": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "DomainEndpoint" + ] + } + ] + ] + } + }, + { + "Name": "RESOURCES_INDEX", + "Value": "some-resources-index" + }, + { + "Name": "BACKEND_URL", + "Value": "some-url.test" + }, + { + "Name": "QUEUE_NAME", + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "QueueName" + ] + } + } + ], + "Essential": true, + "Image": { + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:6bc4c4118fffd7f29f1cc00e6e44c3519d67cd0fa51368b52669d25a0ff287f1" + }, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "IndexerIndexingServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupE85BD506" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "indexing-service", + "mode": "non-blocking" + } + }, + "Name": "QueueProcessingContainer", + "Secrets": [ + { + "Name": "BACKEND_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "TestSecret16AF87B1" + }, + ":BACKEND_KEY::" + ] + ] + } + }, + { + "Name": "BACKEND_SECRET_KEY", + "ValueFrom": { + "Fn::Join": [ + "", + [ + { + "Ref": "TestSecret16AF87B1" + }, + ":BACKEND_SECRET_KEY::" + ] + ] + } + } + ] + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingTaskDefExecutionRoleCFBD9590", + "Arn" + ] + }, + "Family": "IndexerIndexingServiceQueueProcessingTaskDefE0CF9535", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974", + "Arn" + ] + } + }, + "Type": "AWS::ECS::TaskDefinition" + }, + "IndexerIndexingServiceQueueProcessingTaskDefExecutionRoleCFBD9590": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "IndexerIndexingServiceQueueProcessingTaskDefExecutionRoleDefaultPolicyD531DDB0": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ecr:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":repository/", + { + "Fn::Sub": "cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + }, + { + "Action": "ecr:GetAuthorizationToken", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupE85BD506", + "Arn" + ] + } + }, + { + "Action": [ + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + ], + "Effect": "Allow", + "Resource": { + "Ref": "TestSecret16AF87B1" + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "IndexerIndexingServiceQueueProcessingTaskDefExecutionRoleDefaultPolicyD531DDB0", + "Roles": [ + { + "Ref": "IndexerIndexingServiceQueueProcessingTaskDefExecutionRoleCFBD9590" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "IndexerIndexingServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupE85BD506": { + "DeletionPolicy": "Retain", + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain" + }, + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "logs:DescribeLogGroups", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleDefaultPolicy8E32F607", + "Roles": [ + { + "Ref": "IndexerIndexingServiceQueueProcessingTaskDefTaskRoleAFEF8974" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "IndexerInvalidationServiceAlarmsCPUAlarm70127DE5": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClusterName", + "Value": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + } + }, + { + "Name": "ServiceName", + "Value": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingFargateService432F03DD", + "Name" + ] + } + } + ], + "EvaluationPeriods": 2, + "MetricName": "CPUUtilization", + "Namespace": "AWS/ECS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Average", + "Threshold": 95 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerInvalidationServiceAlarmsMemoryAlarm9612A989": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClusterName", + "Value": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + } + }, + { + "Name": "ServiceName", + "Value": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingFargateService432F03DD", + "Name" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "MemoryUtilization", + "Namespace": "AWS/ECS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Average", + "Threshold": 80 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerInvalidationServiceQueueProcessingFargateService432F03DD": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "Cluster": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + }, + "DeploymentConfiguration": { + "Alarms": { + "AlarmNames": [], + "Enable": false, + "Rollback": false + }, + "DeploymentCircuitBreaker": { + "Enable": true, + "Rollback": true + }, + "MaximumPercent": 200, + "MinimumHealthyPercent": 50 + }, + "DeploymentController": { + "Type": "ECS" + }, + "EnableECSManagedTags": false, + "EnableExecuteCommand": true, + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingFargateServiceSecurityGroupB0C75422", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "TestVpcpublicSubnet1Subnet4F70BC85" + }, + { + "Ref": "TestVpcpublicSubnet2Subnet96FF72E6" + } + ] + } + }, + "ServiceName": "InvalidationService", + "TaskDefinition": { + "Ref": "IndexerInvalidationServiceQueueProcessingTaskDefAD9C4E3C" + } + }, + "Type": "AWS::ECS::Service" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceSecurityGroupB0C75422": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "GroupDescription": "Default/Indexer/InvalidationService/QueueProcessingFargateService/SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetE5561481": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "MaxCapacity": 2, + "MinCapacity": 1, + "ResourceId": { + "Fn::Join": [ + "", + [ + "service/", + { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + }, + "/", + { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingFargateService432F03DD", + "Name" + ] + } + ] + ] + }, + "RoleARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/aws-service-role/ecs.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_ECSService" + ] + ] + }, + "ScalableDimension": "ecs:service:DesiredCount", + "ServiceNamespace": "ecs" + }, + "Type": "AWS::ApplicationAutoScaling::ScalableTarget" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerAlarm45916FDF": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "AlarmActions": [ + { + "Ref": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicyAE52ED05" + } + ], + "AlarmDescription": "Lower threshold scaling alarm", + "ComparisonOperator": "LessThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 0 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicyAE52ED05": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "PolicyName": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicyA4356389", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetE5561481" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalUpperBound": 0, + "ScalingAdjustment": -1 + } + ] + } + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperAlarm9B61F437": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "AlarmActions": [ + { + "Ref": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy0F3EFD44" + } + ], + "AlarmDescription": "Upper threshold scaling alarm", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy0F3EFD44": { + "DependsOn": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + ], + "Properties": { + "PolicyName": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy1CC0543C", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetE5561481" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalLowerBound": 0, + "MetricIntervalUpperBound": 999, + "ScalingAdjustment": 1 + }, + { + "MetricIntervalLowerBound": 999, + "ScalingAdjustment": 2 + } + ] + } + }, + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefAD9C4E3C": { + "Properties": { + "ContainerDefinitions": [ + { + "Command": [ + "run-invalidation-service" + ], + "Environment": [ + { + "Name": "OPENSEARCH_URL", + "Value": { + "Fn::Join": [ + "", + [ + "https://", + { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "DomainEndpoint" + ] + } + ] + ] + } + }, + { + "Name": "TRANSACTION_QUEUE_URL", + "Value": { + "Ref": "TransactionQueueE05C979B" + } + }, + { + "Name": "INVALIDATION_QUEUE_URL", + "Value": { + "Ref": "InvalidationQueue8614463D" + } + }, + { + "Name": "RESOURCES_INDEX", + "Value": "some-resources-index" + }, + { + "Name": "QUEUE_NAME", + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "QueueName" + ] + } + } + ], + "Essential": true, + "Image": { + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:6bc4c4118fffd7f29f1cc00e6e44c3519d67cd0fa51368b52669d25a0ff287f1" + }, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "IndexerInvalidationServiceQueueProcessingTaskDefQueueProcessingContainerLogGroup7A19AA4F" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "invalidation-service", + "mode": "non-blocking" + } + }, + "Name": "QueueProcessingContainer" + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingTaskDefExecutionRole7CBAB82A", + "Arn" + ] + }, + "Family": "IndexerInvalidationServiceQueueProcessingTaskDef5AA84E8A", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69", + "Arn" + ] + } + }, + "Type": "AWS::ECS::TaskDefinition" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefExecutionRole7CBAB82A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefExecutionRoleDefaultPolicyEEDD4CA1": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ecr:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":repository/", + { + "Fn::Sub": "cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + }, + { + "Action": "ecr:GetAuthorizationToken", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingTaskDefQueueProcessingContainerLogGroup7A19AA4F", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "IndexerInvalidationServiceQueueProcessingTaskDefExecutionRoleDefaultPolicyEEDD4CA1", + "Roles": [ + { + "Ref": "IndexerInvalidationServiceQueueProcessingTaskDefExecutionRole7CBAB82A" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefQueueProcessingContainerLogGroup7A19AA4F": { + "DeletionPolicy": "Retain", + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "logs:DescribeLogGroups", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:DescribeLogStreams", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "Arn" + ] + } + }, + { + "Action": [ + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "IndexerInvalidationServiceQueueProcessingTaskDefTaskRoleDefaultPolicy6C08482B", + "Roles": [ + { + "Ref": "IndexerInvalidationServiceQueueProcessingTaskDefTaskRole1C46CA69" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "InvalidationQueue8614463D": { + "DeletionPolicy": "Delete", + "Properties": { + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "InvalidationQueueDeadLetterQueueFE5C594E", + "Arn" + ] + }, + "maxReceiveCount": 3 + }, + "VisibilityTimeout": 120 + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete" + }, + "InvalidationQueueDeadLetterQueueFE5C594E": { + "DeletionPolicy": "Delete", + "Properties": { + "MessageRetentionPeriod": 1209600 + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete" + }, + "InvalidationQueueInvalidationQueueAlarmsDeadLetterQueueHasMessagesAlarmFDDAE60C": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "InvalidationQueueDeadLetterQueueFE5C594E", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "InvalidationQueueInvalidationQueueAlarmsQueueHasOldMessagesAlarm11ADDAD4": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "InvalidationQueue8614463D", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateAgeOfOldestMessage", + "Namespace": "AWS/SQS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 3600 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchDomainAccessPolicyCustomResourcePolicyDBAF1B3A": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "es:UpdateDomainConfig", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "OpensearchDomainAccessPolicyCustomResourcePolicyDBAF1B3A", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "OpensearchDomainAccessPolicyD2BDC5BE": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "OpensearchDomainAccessPolicyCustomResourcePolicyDBAF1B3A" + ], + "Properties": { + "Create": { + "Fn::Join": [ + "", + [ + "{\"action\":\"updateDomainConfig\",\"service\":\"OpenSearch\",\"parameters\":{\"DomainName\":\"", + { + "Ref": "OpensearchDomainCED7C974" + }, + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "Arn" + ] + }, + "/*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPaths\":[\"DomainConfig.AccessPolicies\"],\"physicalResourceId\":{\"id\":\"", + { + "Ref": "OpensearchDomainCED7C974" + }, + "AccessPolicy\"}}" + ] + ] + }, + "InstallLatestAwsSdk": true, + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Update": { + "Fn::Join": [ + "", + [ + "{\"action\":\"updateDomainConfig\",\"service\":\"OpenSearch\",\"parameters\":{\"DomainName\":\"", + { + "Ref": "OpensearchDomainCED7C974" + }, + "\",\"AccessPolicies\":\"{\\\"Statement\\\":[{\\\"Action\\\":\\\"es:ESHttp*\\\",\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"AWS\\\":\\\"*\\\"},\\\"Resource\\\":\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainCED7C974", + "Arn" + ] + }, + "/*\\\"}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"outputPaths\":[\"DomainConfig.AccessPolicies\"],\"physicalResourceId\":{\"id\":\"", + { + "Ref": "OpensearchDomainCED7C974" + }, + "AccessPolicy\"}}" + ] + ] + } + }, + "Type": "Custom::OpenSearchAccessPolicy", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchDomainAppLogs191CCADD": { + "DeletionPolicy": "Delete", + "Properties": { + "RetentionInDays": 30, + "Tags": [ + { + "Key": "branch", + "Value": "some-branch" + } + ] + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchDomainCED7C974": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8CustomResourcePolicyA49EBF6B", + "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8897F1FE0" + ], + "Properties": { + "AdvancedOptions": { + "indices.query.bool.max_clause_count": "8096" + }, + "ClusterConfig": { + "DedicatedMasterEnabled": false, + "InstanceCount": 1, + "InstanceType": "t3.small.search", + "ZoneAwarenessEnabled": false + }, + "DomainEndpointOptions": { + "EnforceHTTPS": false, + "TLSSecurityPolicy": "Policy-Min-TLS-1-0-2019-07" + }, + "EBSOptions": { + "EBSEnabled": true, + "VolumeSize": 10, + "VolumeType": "gp2" + }, + "EncryptionAtRestOptions": { + "Enabled": false + }, + "EngineVersion": "OpenSearch_2.3", + "LogPublishingOptions": { + "ES_APPLICATION_LOGS": { + "CloudWatchLogsLogGroupArn": { + "Fn::GetAtt": [ + "OpensearchDomainAppLogs191CCADD", + "Arn" + ] + }, + "Enabled": true + }, + "INDEX_SLOW_LOGS": { + "CloudWatchLogsLogGroupArn": { + "Fn::GetAtt": [ + "OpensearchDomainSlowIndexLogs76621B18", + "Arn" + ] + }, + "Enabled": true + }, + "SEARCH_SLOW_LOGS": { + "CloudWatchLogsLogGroupArn": { + "Fn::GetAtt": [ + "OpensearchDomainSlowSearchLogsCB6C5516", + "Arn" + ] + }, + "Enabled": true + } + }, + "NodeToNodeEncryptionOptions": { + "Enabled": false + }, + "Tags": [ + { + "Key": "branch", + "Value": "some-branch" + } + ], + "VPCOptions": { + "SecurityGroupIds": [ + { + "Fn::GetAtt": [ + "OpensearchDomainSecurityGroup046A436D", + "GroupId" + ] + } + ], + "SubnetIds": [] + } + }, + "Type": "AWS::OpenSearchService::Domain", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8897F1FE0": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8CustomResourcePolicyA49EBF6B" + ], + "Properties": { + "Create": { + "Fn::Join": [ + "", + [ + "{\"service\":\"CloudWatchLogs\",\"action\":\"putResourcePolicy\",\"parameters\":{\"policyName\":\"ESLogPolicyc846790a681c5ad3f799b04b167a965f02d02470a8\",\"policyDocument\":\"{\\\"Statement\\\":[{\\\"Action\\\":[\\\"logs:PutLogEvents\\\",\\\"logs:CreateLogStream\\\"],\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"Service\\\":\\\"es.amazonaws.com\\\"},\\\"Resource\\\":[\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainSlowSearchLogsCB6C5516", + "Arn" + ] + }, + "\\\",\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainSlowIndexLogs76621B18", + "Arn" + ] + }, + "\\\",\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainAppLogs191CCADD", + "Arn" + ] + }, + "\\\"]}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"physicalResourceId\":{\"id\":\"ESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8\"}}" + ] + ] + }, + "Delete": "{\"service\":\"CloudWatchLogs\",\"action\":\"deleteResourcePolicy\",\"parameters\":{\"policyName\":\"ESLogPolicyc846790a681c5ad3f799b04b167a965f02d02470a8\"},\"ignoreErrorCodesMatching\":\"ResourceNotFoundException\"}", + "InstallLatestAwsSdk": true, + "ServiceToken": { + "Fn::GetAtt": [ + "AWS679f53fac002430cb0da5b7982bd22872D164C4C", + "Arn" + ] + }, + "Update": { + "Fn::Join": [ + "", + [ + "{\"service\":\"CloudWatchLogs\",\"action\":\"putResourcePolicy\",\"parameters\":{\"policyName\":\"ESLogPolicyc846790a681c5ad3f799b04b167a965f02d02470a8\",\"policyDocument\":\"{\\\"Statement\\\":[{\\\"Action\\\":[\\\"logs:PutLogEvents\\\",\\\"logs:CreateLogStream\\\"],\\\"Effect\\\":\\\"Allow\\\",\\\"Principal\\\":{\\\"Service\\\":\\\"es.amazonaws.com\\\"},\\\"Resource\\\":[\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainSlowSearchLogsCB6C5516", + "Arn" + ] + }, + "\\\",\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainSlowIndexLogs76621B18", + "Arn" + ] + }, + "\\\",\\\"", + { + "Fn::GetAtt": [ + "OpensearchDomainAppLogs191CCADD", + "Arn" + ] + }, + "\\\"]}],\\\"Version\\\":\\\"2012-10-17\\\"}\"},\"physicalResourceId\":{\"id\":\"ESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8\"}}" + ] + ] + } + }, + "Type": "Custom::CloudwatchLogResourcePolicy", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8CustomResourcePolicyA49EBF6B": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "logs:PutResourcePolicy", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": "logs:DeleteResourcePolicy", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "OpensearchDomainESLogGroupPolicyc846790a681c5ad3f799b04b167a965f02d02470a8CustomResourcePolicyA49EBF6B", + "Roles": [ + { + "Ref": "AWS679f53fac002430cb0da5b7982bd2287ServiceRoleC1EA0FF2" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "OpensearchDomainSecurityGroup046A436D": { + "Properties": { + "GroupDescription": "Security group for domain Domain", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "Tags": [ + { + "Key": "branch", + "Value": "some-branch" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "OpensearchDomainSecurityGroupfromIndexerIndexingServiceQueueProcessingFargateServiceSecurityGroupE48AFA19443A85AE460": { + "Properties": { + "Description": "Allow connection to Opensearch", + "FromPort": 443, + "GroupId": { + "Fn::GetAtt": [ + "OpensearchDomainSecurityGroup046A436D", + "GroupId" + ] + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "IndexerIndexingServiceQueueProcessingFargateServiceSecurityGroup9D88A792", + "GroupId" + ] + }, + "ToPort": 443 + }, + "Type": "AWS::EC2::SecurityGroupIngress" + }, + "OpensearchDomainSecurityGroupfromIndexerInvalidationServiceQueueProcessingFargateServiceSecurityGroupA65071664430D51B66B": { + "Properties": { + "Description": "Allow connection to Opensearch", + "FromPort": 443, + "GroupId": { + "Fn::GetAtt": [ + "OpensearchDomainSecurityGroup046A436D", + "GroupId" + ] + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "IndexerInvalidationServiceQueueProcessingFargateServiceSecurityGroupB0C75422", + "GroupId" + ] + }, + "ToPort": 443 + }, + "Type": "AWS::EC2::SecurityGroupIngress" + }, + "OpensearchDomainSlowIndexLogs76621B18": { + "DeletionPolicy": "Delete", + "Properties": { + "RetentionInDays": 30, + "Tags": [ + { + "Key": "branch", + "Value": "some-branch" + } + ] + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchDomainSlowSearchLogsCB6C5516": { + "DeletionPolicy": "Delete", + "Properties": { + "RetentionInDays": 30, + "Tags": [ + { + "Key": "branch", + "Value": "some-branch" + } + ] + }, + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Delete" + }, + "OpensearchOpensearchAlarmsCPUAlarm33CD9214": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "CPUUtilization", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 85 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchOpensearchAlarmsClusterBlockingWritesAlarmF52E6F69": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ClusterIndexWritesBlocked", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 60, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchOpensearchAlarmsClusterStatusRedAlarm74B637B9": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ClusterStatus.red", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchOpensearchAlarmsJVMMemoryAlarm33877A8F": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "JVMMemoryPressure", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 90 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchOpensearchAlarmsSearchLatencyAlarm83BCB0F0": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "ExtendedStatistic": "p99", + "MetricName": "SearchLatency", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Threshold": 1000 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "OpensearchOpensearchAlarmsStorageAlarm22102EA9": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "LessThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "ClientId", + "Value": { + "Ref": "AWS::AccountId" + } + }, + { + "Name": "DomainName", + "Value": { + "Ref": "OpensearchDomainCED7C974" + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "FreeStorageSpace", + "Namespace": "AWS/ES", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Minimum", + "Threshold": 2560 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "TestApplicationLoadBalancedFargateService8543A9A0": { + "DependsOn": [ + "TestApplicationLoadBalancedFargateServiceLBPublicListenerECSGroup6D891061", + "TestApplicationLoadBalancedFargateServiceLBPublicListener241CC671", + "TestApplicationLoadBalancedFargateServiceTaskDefTaskRole3FDBA2D6" + ], + "Properties": { + "Cluster": { + "Ref": "EcsDefaultClusterMnL3mNNYNTestVpc4872C696" + }, + "DeploymentConfiguration": { + "Alarms": { + "AlarmNames": [], + "Enable": false, + "Rollback": false + }, + "MaximumPercent": 200, + "MinimumHealthyPercent": 50 + }, + "EnableECSManagedTags": false, + "HealthCheckGracePeriodSeconds": 60, + "LaunchType": "FARGATE", + "LoadBalancers": [ + { + "ContainerName": "web", + "ContainerPort": 80, + "TargetGroupArn": { + "Ref": "TestApplicationLoadBalancedFargateServiceLBPublicListenerECSGroup6D891061" + } + } + ], + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "ENABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceSecurityGroup7B60D67E", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "TestVpcpublicSubnet1Subnet4F70BC85" + }, + { + "Ref": "TestVpcpublicSubnet2Subnet96FF72E6" + } + ] + } + }, + "TaskDefinition": { + "Ref": "TestApplicationLoadBalancedFargateServiceTaskDefBBD234AC" + } + }, + "Type": "AWS::ECS::Service" + }, + "TestApplicationLoadBalancedFargateServiceLBC13F6669": { + "DependsOn": [ + "TestVpcpublicSubnet1DefaultRoute5A98368D", + "TestVpcpublicSubnet1RouteTableAssociationEA0572FC", + "TestVpcpublicSubnet2DefaultRouteBF011106", + "TestVpcpublicSubnet2RouteTableAssociation8BC8AAA7" + ], + "Properties": { + "LoadBalancerAttributes": [ + { + "Key": "deletion_protection.enabled", + "Value": "false" + } + ], + "Scheme": "internet-facing", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceLBSecurityGroup0971E8F2", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "TestVpcpublicSubnet1Subnet4F70BC85" + }, + { + "Ref": "TestVpcpublicSubnet2Subnet96FF72E6" + } + ], + "Type": "application" + }, + "Type": "AWS::ElasticLoadBalancingV2::LoadBalancer" + }, + "TestApplicationLoadBalancedFargateServiceLBPublicListener241CC671": { + "Properties": { + "DefaultActions": [ + { + "TargetGroupArn": { + "Ref": "TestApplicationLoadBalancedFargateServiceLBPublicListenerECSGroup6D891061" + }, + "Type": "forward" + } + ], + "LoadBalancerArn": { + "Ref": "TestApplicationLoadBalancedFargateServiceLBC13F6669" + }, + "Port": 80, + "Protocol": "HTTP" + }, + "Type": "AWS::ElasticLoadBalancingV2::Listener" + }, + "TestApplicationLoadBalancedFargateServiceLBPublicListenerECSGroup6D891061": { + "Properties": { + "Port": 80, + "Protocol": "HTTP", + "TargetGroupAttributes": [ + { + "Key": "stickiness.enabled", + "Value": "false" + } + ], + "TargetType": "ip", + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::ElasticLoadBalancingV2::TargetGroup" + }, + "TestApplicationLoadBalancedFargateServiceLBSecurityGroup0971E8F2": { + "Properties": { + "GroupDescription": "Automatically created Security Group for ELB TestApplicationLoadBalancedFargateServiceLBA215F647", + "SecurityGroupIngress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow from anyone on port 80", + "FromPort": 80, + "IpProtocol": "tcp", + "ToPort": 80 + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "TestApplicationLoadBalancedFargateServiceLBSecurityGrouptoTestApplicationLoadBalancedFargateServiceSecurityGroup07BD40948022E83761": { + "Properties": { + "Description": "Load balancer to target", + "DestinationSecurityGroupId": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceSecurityGroup7B60D67E", + "GroupId" + ] + }, + "FromPort": 80, + "GroupId": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceLBSecurityGroup0971E8F2", + "GroupId" + ] + }, + "IpProtocol": "tcp", + "ToPort": 80 + }, + "Type": "AWS::EC2::SecurityGroupEgress" + }, + "TestApplicationLoadBalancedFargateServiceSecurityGroup7B60D67E": { + "DependsOn": [ + "TestApplicationLoadBalancedFargateServiceTaskDefTaskRole3FDBA2D6" + ], + "Properties": { + "GroupDescription": "Default/TestApplicationLoadBalancedFargateService/Service/SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::SecurityGroup" + }, + "TestApplicationLoadBalancedFargateServiceSecurityGroupfromTestApplicationLoadBalancedFargateServiceLBSecurityGroup8A5A22A480EA1AFA65": { + "DependsOn": [ + "TestApplicationLoadBalancedFargateServiceTaskDefTaskRole3FDBA2D6" + ], + "Properties": { + "Description": "Load balancer to target", + "FromPort": 80, + "GroupId": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceSecurityGroup7B60D67E", + "GroupId" + ] + }, + "IpProtocol": "tcp", + "SourceSecurityGroupId": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceLBSecurityGroup0971E8F2", + "GroupId" + ] + }, + "ToPort": 80 + }, + "Type": "AWS::EC2::SecurityGroupIngress" + }, + "TestApplicationLoadBalancedFargateServiceTaskDefBBD234AC": { + "Properties": { + "ContainerDefinitions": [ + { + "Essential": true, + "Image": "some-test-image", + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "TestApplicationLoadBalancedFargateServiceTaskDefwebLogGroup1916795D" + }, + "awslogs-region": { + "Ref": "AWS::Region" + }, + "awslogs-stream-prefix": "TestApplicationLoadBalancedFargateService" + } + }, + "Name": "web", + "PortMappings": [ + { + "ContainerPort": 80, + "Protocol": "tcp" + } + ] + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceTaskDefExecutionRole9F745D1D", + "Arn" + ] + }, + "Family": "TestApplicationLoadBalancedFargateServiceTaskDef0A28FBC0", + "Memory": "512", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ], + "TaskRoleArn": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceTaskDefTaskRole3FDBA2D6", + "Arn" + ] + } + }, + "Type": "AWS::ECS::TaskDefinition" + }, + "TestApplicationLoadBalancedFargateServiceTaskDefExecutionRole9F745D1D": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "TestApplicationLoadBalancedFargateServiceTaskDefExecutionRoleDefaultPolicy54989FA8": { + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "TestApplicationLoadBalancedFargateServiceTaskDefwebLogGroup1916795D", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "TestApplicationLoadBalancedFargateServiceTaskDefExecutionRoleDefaultPolicy54989FA8", + "Roles": [ + { + "Ref": "TestApplicationLoadBalancedFargateServiceTaskDefExecutionRole9F745D1D" + } + ] + }, + "Type": "AWS::IAM::Policy" + }, + "TestApplicationLoadBalancedFargateServiceTaskDefTaskRole3FDBA2D6": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "TestApplicationLoadBalancedFargateServiceTaskDefwebLogGroup1916795D": { + "DeletionPolicy": "Retain", + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain" + }, + "TestBusF2C65FE8": { + "Properties": { + "Name": "TestBus" + }, + "Type": "AWS::Events::EventBus" + }, + "TestCertificate6B4956B6": { + "Properties": { + "DomainName": "*.my.test.domain.org", + "DomainValidationOptions": [ + { + "DomainName": "*.my.test.domain.org", + "ValidationDomain": "domain.org" + } + ], + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestCertificate" + } + ], + "ValidationMethod": "EMAIL" + }, + "Type": "AWS::CertificateManager::Certificate" + }, + "TestChatbot57319657": { + "Properties": { + "ConfigurationName": "some-config-name", + "IamRoleArn": { + "Fn::GetAtt": [ + "TestChatbotConfigurationRole238BFAFD", + "Arn" + ] + }, + "SlackChannelId": "some-channel-id", + "SlackWorkspaceId": "some-workspace-id" + }, + "Type": "AWS::Chatbot::SlackChannelConfiguration" + }, + "TestChatbotConfigurationRole238BFAFD": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "chatbot.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::Role" + }, + "TestHostedZone68F306E4": { + "Properties": { + "Name": "my.test.domain.org." + }, + "Type": "AWS::Route53::HostedZone" + }, + "TestSecret16AF87B1": { + "DeletionPolicy": "Delete", + "Properties": { + "GenerateSecretString": {} + }, + "Type": "AWS::SecretsManager::Secret", + "UpdateReplacePolicy": "Delete" + }, + "TestTopic339EC197": { + "Type": "AWS::SNS::Topic" + }, + "TestVpcE77CE678": { + "Properties": { + "CidrBlock": "10.4.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc" + } + ] + }, + "Type": "AWS::EC2::VPC" + }, + "TestVpcIGW9DD53F70": { + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc" + } + ] + }, + "Type": "AWS::EC2::InternetGateway" + }, + "TestVpcVPCGWF1827B84": { + "Properties": { + "InternetGatewayId": { + "Ref": "TestVpcIGW9DD53F70" + }, + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::VPCGatewayAttachment" + }, + "TestVpcisolatedSubnet1RouteTable93EFC1F5": { + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc/isolatedSubnet1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::RouteTable" + }, + "TestVpcisolatedSubnet1RouteTableAssociationB18A00D7": { + "Properties": { + "RouteTableId": { + "Ref": "TestVpcisolatedSubnet1RouteTable93EFC1F5" + }, + "SubnetId": { + "Ref": "TestVpcisolatedSubnet1Subnet2860A680" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "TestVpcisolatedSubnet1Subnet2860A680": { + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 0, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.4.32.0/20", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "isolated" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Isolated" + }, + { + "Key": "Name", + "Value": "Default/TestVpc/isolatedSubnet1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "TestVpcisolatedSubnet2RouteTable7EBE60D9": { + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc/isolatedSubnet2" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::RouteTable" + }, + "TestVpcisolatedSubnet2RouteTableAssociation2BB4538E": { + "Properties": { + "RouteTableId": { + "Ref": "TestVpcisolatedSubnet2RouteTable7EBE60D9" + }, + "SubnetId": { + "Ref": "TestVpcisolatedSubnet2SubnetA6454F0B" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "TestVpcisolatedSubnet2SubnetA6454F0B": { + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 1, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.4.48.0/20", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "isolated" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Isolated" + }, + { + "Key": "Name", + "Value": "Default/TestVpc/isolatedSubnet2" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "TestVpcpublicSubnet1DefaultRoute5A98368D": { + "DependsOn": [ + "TestVpcVPCGWF1827B84" + ], + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "TestVpcIGW9DD53F70" + }, + "RouteTableId": { + "Ref": "TestVpcpublicSubnet1RouteTableD55D3AC3" + } + }, + "Type": "AWS::EC2::Route" + }, + "TestVpcpublicSubnet1RouteTableAssociationEA0572FC": { + "Properties": { + "RouteTableId": { + "Ref": "TestVpcpublicSubnet1RouteTableD55D3AC3" + }, + "SubnetId": { + "Ref": "TestVpcpublicSubnet1Subnet4F70BC85" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "TestVpcpublicSubnet1RouteTableD55D3AC3": { + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc/publicSubnet1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::RouteTable" + }, + "TestVpcpublicSubnet1Subnet4F70BC85": { + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 0, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.4.0.0/20", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "Default/TestVpc/publicSubnet1" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "TestVpcpublicSubnet2DefaultRouteBF011106": { + "DependsOn": [ + "TestVpcVPCGWF1827B84" + ], + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "TestVpcIGW9DD53F70" + }, + "RouteTableId": { + "Ref": "TestVpcpublicSubnet2RouteTable19CD621C" + } + }, + "Type": "AWS::EC2::Route" + }, + "TestVpcpublicSubnet2RouteTable19CD621C": { + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "Default/TestVpc/publicSubnet2" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::RouteTable" + }, + "TestVpcpublicSubnet2RouteTableAssociation8BC8AAA7": { + "Properties": { + "RouteTableId": { + "Ref": "TestVpcpublicSubnet2RouteTable19CD621C" + }, + "SubnetId": { + "Ref": "TestVpcpublicSubnet2Subnet96FF72E6" + } + }, + "Type": "AWS::EC2::SubnetRouteTableAssociation" + }, + "TestVpcpublicSubnet2Subnet96FF72E6": { + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 1, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.4.16.0/20", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "Default/TestVpc/publicSubnet2" + } + ], + "VpcId": { + "Ref": "TestVpcE77CE678" + } + }, + "Type": "AWS::EC2::Subnet" + }, + "TransactionQueueDeadLetterQueueDA53F160": { + "DeletionPolicy": "Delete", + "Properties": { + "MessageRetentionPeriod": 1209600 + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete" + }, + "TransactionQueueE05C979B": { + "DeletionPolicy": "Delete", + "Properties": { + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "TransactionQueueDeadLetterQueueDA53F160", + "Arn" + ] + }, + "maxReceiveCount": 3 + }, + "VisibilityTimeout": 120 + }, + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete" + }, + "TransactionQueueTransactionQueueAlarmsDeadLetterQueueHasMessagesAlarm3FEEE842": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "TransactionQueueDeadLetterQueueDA53F160", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "TransactionQueueTransactionQueueAlarmsQueueHasOldMessagesAlarm368551C1": { + "Properties": { + "AlarmActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "TransactionQueueE05C979B", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateAgeOfOldestMessage", + "Namespace": "AWS/SQS", + "OKActions": [ + { + "Ref": "TestTopic339EC197" + } + ], + "Period": 300, + "Statistic": "Maximum", + "Threshold": 3600 + }, + "Type": "AWS::CloudWatch::Alarm" + }, + "UploadManagedPolicy7658189E": { + "Properties": { + "Description": "", + "Path": "/", + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:PutObject", + "Effect": "Allow", + "Resource": "arn:aws:s3:::some-test-bucket/" + } + ], + "Version": "2012-10-17" + } + }, + "Type": "AWS::IAM::ManagedPolicy" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/cdk/tests/unit/test_constructs_indexer.py b/cdk/tests/unit/test_constructs_indexer.py index c08cb4749..da0ad3dae 100644 --- a/cdk/tests/unit/test_constructs_indexer.py +++ b/cdk/tests/unit/test_constructs_indexer.py @@ -423,9 +423,91 @@ def test_constructs_indexer_initialize_indexer( ) template.resource_count_is( 'AWS::ApplicationAutoScaling::ScalingPolicy', - 6 + 4 ) template.resource_count_is( 'AWS::CloudWatch::Alarm', 18 ) + cpu_scaling_resources = template.find_resources( + 'AWS::ApplicationAutoScaling::ScalingPolicy', + { + 'Properties': { + 'PolicyType': 'TargetTrackingScaling', + 'TargetTrackingScalingPolicyConfiguration': { + 'PredefinedMetricSpecification': { + 'PredefinedMetricType': 'ECSServiceAverageCPUUtilization' + }, + 'TargetValue': 50 + } + } + } + ) + assert len(cpu_scaling_resources) == 0 + template.has_resource_properties( + 'AWS::ApplicationAutoScaling::ScalingPolicy', + { + 'PolicyName': 'IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicyA4356389', + 'PolicyType': 'StepScaling', + 'ScalingTargetId': { + 'Ref': 'IndexerInvalidationServiceQueueProcessingFargateServiceTaskCountTargetE5561481' + }, + 'StepScalingPolicyConfiguration': { + 'AdjustmentType': 'ChangeInCapacity', + 'MetricAggregationType': 'Maximum', + 'StepAdjustments': [ + { + 'MetricIntervalUpperBound': 0, + 'ScalingAdjustment': -1 + } + ] + } + } + ) + + +def test_constructs_indexer_indexer_match_snapshot( + stack, + config, + existing_resources, + application_load_balanced_fargate_service, + transaction_queue, + invalidation_queue, + opensearch_multiplexer, + snapshot, +): + import json + from infrastructure.constructs.indexer import InvalidationServiceProps + from infrastructure.constructs.indexer import IndexingServiceProps + from infrastructure.constructs.indexer import IndexerProps + from infrastructure.constructs.indexer import Indexer + indexer = Indexer( + stack, + 'Indexer', + props=IndexerProps( + config=config, + existing_resources=existing_resources, + cluster=application_load_balanced_fargate_service.cluster, + transaction_queue=transaction_queue, + invalidation_queue=invalidation_queue, + opensearch_multiplexer=opensearch_multiplexer, + use_opensearch_named='Opensearch', + backend_url='some-url.test', + resources_index='some-resources-index', + invalidation_service_props=InvalidationServiceProps( + **config.invalidation_service, + ), + indexing_service_props=IndexingServiceProps( + **config.indexing_service, + ) + ) + ) + template = Template.from_stack(stack) + snapshot.assert_match( + json.dumps( + template.to_json(), + indent=4, + sort_keys=True + ), + 'indexer_constructs_template.json' + ) diff --git a/setup.cfg b/setup.cfg index 9fea29dea..7ca51fa12 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ install_requires = botocore==1.29.27 certifi==2023.7.22 future==0.18.2 + google-cloud-storage==2.13.0 gunicorn==20.1.0 humanfriendly==6.1 jsonschema[format]==4.4.0 diff --git a/src/igvfd/tests/test_upload_credentials.py b/src/igvfd/tests/test_upload_credentials.py index 9af893e24..e40bf1ea0 100644 --- a/src/igvfd/tests/test_upload_credentials.py +++ b/src/igvfd/tests/test_upload_credentials.py @@ -2,10 +2,10 @@ import os -def test_upload_credentials_init_upload_credentials(): +def test_upload_credentials_init_s3_upload_credentials(): from igvfd.upload_credentials import get_sts_client - from igvfd.upload_credentials import UploadCredentials - upload_credentials = UploadCredentials( + from igvfd.upload_credentials import S3UploadCredentials + upload_credentials = S3UploadCredentials( bucket='some-bucket', key='some-key', name='some-name', @@ -15,13 +15,17 @@ def test_upload_credentials_init_upload_credentials(): ) ) ) - assert isinstance(upload_credentials, UploadCredentials) + assert isinstance(upload_credentials, S3UploadCredentials) -def test_upload_credentials_upload_credentials_external_credentials(): +def test_upload_credentials_init_gcs_upload_credentials(): + pass + + +def test_upload_credentials_s3_upload_credentials_external_credentials(): from igvfd.upload_credentials import get_sts_client - from igvfd.upload_credentials import UploadCredentials - upload_credentials = UploadCredentials( + from igvfd.upload_credentials import S3UploadCredentials + upload_credentials = S3UploadCredentials( bucket='some-bucket', key='some-key', name='some-name', @@ -53,3 +57,7 @@ def test_upload_credentials_upload_credentials_external_credentials(): for key in expected['upload_credentials']: assert key in actual['upload_credentials'] assert actual['upload_credentials']['upload_url'] == 's3://some-bucket/some-key' + + +def test_upload_credentials_gcs_upload_credentials_external_credentials(): + pass diff --git a/src/igvfd/types/file.py b/src/igvfd/types/file.py index 2d6e3accd..54b5cffb9 100644 --- a/src/igvfd/types/file.py +++ b/src/igvfd/types/file.py @@ -32,9 +32,12 @@ paths_filtered_by_status ) +from igvfd.upload_credentials import get_gcs_client +from igvfd.upload_credentials import get_gcp_credentials from igvfd.upload_credentials import get_s3_client from igvfd.upload_credentials import get_sts_client -from igvfd.upload_credentials import UploadCredentials +from igvfd.upload_credentials import GcsUploadCredentials +from igvfd.upload_credentials import S3UploadCredentials FILE_FORMAT_TO_FILE_EXTENSION = { @@ -162,16 +165,26 @@ def create(cls, registry, uuid, properties, sheets=None): key = f'{date}/{uuid}/{accession}{file_extension}' name = f'up{time.time():.6f}-{accession}'[:32] # max 32 chars profile_name = registry.settings.get('file_upload_profile_name') - upload_credentials = UploadCredentials( - bucket=bucket, - key=key, - name=name, - sts_client=get_sts_client( - localstack_endpoint_url=os.environ.get( - 'LOCALSTACK_ENDPOINT_URL' - ) - ), - ) + + if properties.get('upload_storage_service') == 'gcs': + upload_credentials = GcsUploadCredentials( + bucket=bucket, + key=key, + name=name, + gcs_client=get_gcs_client(), + gcp_credentials=get_gcp_credentials(), + ) + else: + upload_credentials = S3UploadCredentials( + bucket=bucket, + key=key, + name=name, + sts_client=get_sts_client( + localstack_endpoint_url=os.environ.get( + 'LOCALSTACK_ENDPOINT_URL' + ) + ), + ) sheets['external'] = upload_credentials.external_creds() return super(File, cls).create(registry, uuid, properties, sheets) @@ -504,10 +517,15 @@ def get_upload(context, request): external = context.propsheets.get('external', {}) upload_credentials = external.get('upload_credentials') # Show s3 location info for files originally submitted to EDW. - if upload_credentials is None and external.get('service') == 's3': - upload_credentials = { - 'upload_url': 's3://{bucket}/{key}'.format(**external), - } + if upload_credentials is None: + if external.get('service') == 's3': + upload_credentials = { + 'upload_url': 's3://{bucket}/{key}'.format(**external), + } + elif external.get('service') == 'gcs': + upload_credentials = { + 'upload_url': 'gs://{bucket}/{key}'.format(**external), + } return { '@graph': [ { @@ -535,13 +553,14 @@ def post_upload(context, request): properties = context.upgrade_properties() if properties['upload_status'] == 'validated': raise HTTPForbidden( - 'Unable to issue new credentials when uploading_status is validated' + 'Unable to issue new credentials or presigned URL when ' + 'uploading_status is validated' ) external = context.propsheets.get( 'external', {} ) - if external.get('service') == 's3': + if external.get('service') in ['s3', 'gcs']: bucket = external['bucket'] key = external['key'] else: @@ -552,16 +571,27 @@ def post_upload(context, request): name = f'up{time.time():.6f}-{accession}'[:32] # max 32 chars file_upload_bucket = request.registry.settings['file_upload_bucket'] profile_name = request.registry.settings.get('file_upload_profile_name') - upload_credentials = UploadCredentials( - bucket=bucket, - key=key, - name=name, - sts_client=get_sts_client( - localstack_endpoint_url=os.environ.get( - 'LOCALSTACK_ENDPOINT_URL' - ) - ), - ) + + if external.get('service') == 's3': + upload_credentials = S3UploadCredentials( + bucket=bucket, + key=key, + name=name, + sts_client=get_sts_client( + localstack_endpoint_url=os.environ.get( + 'LOCALSTACK_ENDPOINT_URL' + ) + ), + ) + elif external.get('service') == 'gcs': + upload_credentials = GcsUploadCredentials( + bucket=bucket, + key=key, + name=name, + gcs_client=get_gcs_client(), + gcp_credentials=get_gcp_credentials(), + ) + external_credentials = upload_credentials.external_creds() new_properties = None if properties['upload_status'] != 'pending': diff --git a/src/igvfd/upload_credentials.py b/src/igvfd/upload_credentials.py index 94ece43ec..f16d42577 100644 --- a/src/igvfd/upload_credentials.py +++ b/src/igvfd/upload_credentials.py @@ -1,4 +1,5 @@ import copy +import datetime import json import os @@ -7,6 +8,13 @@ from botocore.client import BaseClient +from datetime import timedelta +from datetime import timezone + +from google.api_core.exceptions import ClientError +from google.cloud import storage +from google.oauth2.service_account import Credentials + from typing import Optional @@ -57,6 +65,31 @@ def get_s3_client(localstack_endpoint_url: Optional[str] = None) -> BaseClient: ) +def get_gcp_project(): + upload_files_user_keys = get_upload_files_user_access_key_and_secret_access_key() + return upload_files_user_keys['GCP_PROJECT'] + + +def get_gcp_service_account_json(): + upload_files_user_keys = get_upload_files_user_access_key_and_secret_access_key() + return json.loads( + upload_files_user_keys['GCP_SERVICE_ACCOUNT_JSON_STR'] + ) + + +def get_gcp_credentials(): + return Credentials.from_service_account_info( + get_gcp_service_account_json() + ) + + +def get_gcs_client(): + return storage.Client( + project=get_gcp_project(), + credentials=get_gcp_credentials(), + ) + + EXTERNAL_BUCKET_STATEMENTS = [ { 'Action': 's3:GetObject', @@ -120,7 +153,7 @@ def _get_external_bucket_policy(file_path): return None -class UploadCredentials(object): +class S3UploadCredentials(object): # pylint: disable=too-few-public-methods ''' Build and distribute federate aws credentials for submitting files @@ -208,3 +241,67 @@ def external_creds(self, s3_transfer_allow=False, s3_transfer_buckets=None): 'key': self._key, 'upload_credentials': credentials, } + + +class GcsUploadCredentials(object): + # pylint: disable=too-few-public-methods + ''' + Build and distribute presigned URLs (https://) for submitting files + ''' + + def __init__(self, bucket, key, name, gcs_client, gcp_credentials): + self._bucket = bucket + self._key = key + self._name = name + self._gcs_client = gcs_client + self._file_url = 'gs://{bucket}/{key}'.format( + bucket=self._bucket, + key=self._key + ) + self._gcp_credentials = gcp_credentials + + def _get_presigned_url(self, file_url, expiration_sec): + ''' + Used to generate a presigned URL for PUT method. + ''' + try: + bucket_obj = self._gcs_client.get_bucket(self._bucket) + blob = storage.Blob(name=self._key, bucket=bucket_obj) + + return blob.generate_signed_url( + version='V4', + method='PUT', + expiration=timedelta(seconds=self._expiration_sec), + credentials=self._gcp_credentials, + ) + + except ClientError as ecp: + print('Warning: ', ecp) + return None + + def external_creds(self): + ''' + Used to get the presigned URL. + Such URL expires in _FEDERATION_TOKEN_DURATION_SECONDS. + ''' + expiration_sec = _FEDERATION_TOKEN_DURATION_SECONDS + + upload_url = self._get_presigned_url( + file_url=self._file_url, + expiration_sec=expiration_sec, + ) + + expiration = ( + datetime.now(timezone.utc) + timedelta(seconds=expiration_sec) + ).isoformat() + + credentials = { + 'expiration': expiration, + 'upload_url': upload_url, + } + return { + 'service': 'gcs', + 'bucket': self._bucket, + 'key': self._key, + 'upload_credentials': credentials, + }