diff --git a/tests/ci/cdk/cdk/components.py b/tests/ci/cdk/cdk/components.py index 8b1b9a4b57..dd4a7bfab5 100644 --- a/tests/ci/cdk/cdk/components.py +++ b/tests/ci/cdk/cdk/components.py @@ -59,6 +59,13 @@ def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject, ec "ec2:DescribeInstances", ], resources=["*"])) + lambda_function.add_to_role_policy( + iam.PolicyStatement(effect=iam.Effect.ALLOW, + actions=[ + "ssm:ListDocuments", + "ssm:DeleteDocument", + ], + resources=["arn:aws:ssm:{}:{}:*".format(AWS_REGION, AWS_ACCOUNT)])) events.Rule(scope=self, id="PurgeEventRule", diff --git a/tests/ci/lambda/Cargo.lock b/tests/ci/lambda/Cargo.lock index 8fad9b4bd7..5141e6af9e 100644 --- a/tests/ci/lambda/Cargo.lock +++ b/tests/ci/lambda/Cargo.lock @@ -118,9 +118,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16838e6c9e12125face1c1eff1343c75e3ff540de98ff7ebd61874a89bcfeb9" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -136,6 +136,7 @@ dependencies = [ "aws-sdk-codebuild", "aws-sdk-ec2", "aws-sdk-secretsmanager", + "aws-sdk-ssm", "env_logger", "lambda_runtime", "log", @@ -147,14 +148,15 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f42c2d4218de4dcd890a109461e2f799a1a2ba3bcd2cde9af88360f5df9266c6" +checksum = "a10d5c055aa540164d9561a0e2e74ad30f0dcf7393c3a92f6733ddf9c5762468" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -238,6 +240,29 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sdk-ssm" +version = "1.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27d420415f7f7ab7eef079a108b01fa7b1fdf2a27b2e1631e3c6bbb408d649e" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.12", + "once_cell", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sso" version = "1.39.0" @@ -307,9 +332,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df1b0fa6be58efe9d4ccc257df0a53b89cd8909e86591a13ca54817c87517be" +checksum = "cc8db6904450bafe7473c6ca9123f88cc11089e41a025408f992db4e22d3be68" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -341,9 +366,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.9" +version = "0.60.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9cd0ae3d97daa0a2bf377a4d8e8e1362cae590c4a1aad0d40058ebca18eb91e" +checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -380,9 +405,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.6.3" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0abbf454960d0db2ad12684a1640120e7557294b0ff8e2f11236290a1b293225" +checksum = "a065c0fe6fdbdf9f11817eb68582b2ab4aff9e9c39e986ae48f7ec576c6322db" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -424,9 +449,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.2" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cee7cadb433c781d3299b916fbf620fea813bf38f49db282fb6858141a05cc8" +checksum = "147100a7bea70fa20ef224a6bad700358305f5dc0f84649c53769761395b355b" dependencies = [ "base64-simd", "bytes", diff --git a/tests/ci/lambda/Cargo.toml b/tests/ci/lambda/Cargo.toml index 8cc31631d7..958c16905a 100644 --- a/tests/ci/lambda/Cargo.toml +++ b/tests/ci/lambda/Cargo.toml @@ -10,6 +10,7 @@ publish = false aws-config = "1.1.7" aws-sdk-codebuild = "1.44.0" aws-sdk-ec2 = "1.59.0" +aws-sdk-ssm = "1.51.0" aws-sdk-secretsmanager = "1.39.0" lambda_runtime = "0.8.0" log = "0.4.17" diff --git a/tests/ci/lambda/src/bin/purge-stale-builds.rs b/tests/ci/lambda/src/bin/purge-stale-builds.rs index ede2845b7e..ab6ec4c327 100644 --- a/tests/ci/lambda/src/bin/purge-stale-builds.rs +++ b/tests/ci/lambda/src/bin/purge-stale-builds.rs @@ -5,6 +5,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use aws_sdk_codebuild::types::BuildBatchFilter; use aws_sdk_ec2::operation::describe_instances::DescribeInstancesOutput; use aws_sdk_ec2::types::Filter; +use aws_sdk_ssm::types::DocumentKeyValuesFilter; use lambda_runtime::{service_fn, Error, LambdaEvent}; use serde_json::{json, Value}; @@ -33,7 +34,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { std::env::var("GITHUB_TOKEN_SECRET_NAME") .map_err(|_| "failed to find github access token secret name")?, ) - .await?; + .await?; let github = octocrab::initialise(octocrab::Octocrab::builder().personal_token(github_token)) .map_err(|e| format!("failed to build github client: {e}"))?; @@ -42,7 +43,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { let project = std::env::var("CODEBUILD_PROJECT_NAME").unwrap(); - let have_ec2_permissions: bool = project == "aws-lc-ci-ec2-test-framework"; + let is_ec2_test_framework: bool = project == "aws-lc-ci-ec2-test-framework"; let github_repo_owner = std::env::var("GITHUB_REPO_OWNER").unwrap(); @@ -50,9 +51,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { log::info!("Pulling builds for {project}"); - let builds = get_project_build_batches(&codebuild_client, project.clone()) - .await - .unwrap(); + let builds = get_project_build_batches(&codebuild_client, project.clone()).await?; let project_pull_requests = gather_pull_request_builds(&codebuild_client, builds).await?; @@ -67,12 +66,18 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { .or_insert(builds); } - // Filters for aws-lc-ec2-test-framework specific hosts. - let ec2_client_optional: Option = if have_ec2_permissions { + let ec2_client_optional: Option = if is_ec2_test_framework { Some(aws_sdk_ec2::Client::new(&config)) } else { None }; + let ssm_client_optional: Option = if is_ec2_test_framework { + Some(aws_sdk_ssm::Client::new(&config)) + } else { + None + }; + + // Filters for aws-lc-ec2-test-framework specific hosts. let ec2_framework_filters = vec![ Filter::builder() .name("instance-state-name") @@ -106,6 +111,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { None }; + let mut ssm_deleted_documents: Vec = vec![]; let mut ec2_terminated_instances: Vec = vec![]; let mut stopped_builds: u64 = 0; @@ -137,6 +143,7 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { .map_err(|e| format!("failed to stop_build_batch: {e}"))?; stopped_builds += 1; + // Gather a list of all unneeded ec2 instances and terminate after the loop. if let Some(ref ec2_describe_response) = ec2_describe_response_optional { // Prune unneeded ec2 instances. for reservation in ec2_describe_response.reservations() { @@ -154,6 +161,10 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { } } } + + // Gather a list of old commits. The SSM documents should have the commits within + // their document name. + ssm_deleted_documents.push(old_commit.to_string()); } } @@ -169,6 +180,28 @@ async fn handle(_event: LambdaEvent) -> Result<(), Error> { } } + if !ssm_deleted_documents.is_empty() && is_ec2_test_framework { + log::info!("Query for list of documents to delete with: {:?}",ssm_deleted_documents); + + let all_documents = get_ssm_document_list( + &ssm_client_optional, + ssm_deleted_documents.clone(), + ).await?; + + // Prune hanging ssm documents corresponding to commits. + for document in all_documents { + log::info!("SSM document {:?} will be deleted", document); + if let Some(ref ssm_client) = ssm_client_optional { + ssm_client + .delete_document() + .name(document) + .send() + .await + .map_err(|e| format!("failed to delete ssm document: {e}"))?; + } + } + } + let timestamp = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() @@ -337,3 +370,46 @@ async fn get_project_build_batches( Ok(builds) } + +async fn get_ssm_document_list( + client: &Option, + ssm_deleted_documents: Vec, +) -> Result, String> { + let ssm_client_filters = vec![ + DocumentKeyValuesFilter::builder() + .key("SearchKeyword") + .set_values(Some(ssm_deleted_documents)) + .build(), + DocumentKeyValuesFilter::builder() + .key("Owner") + .values("Self") + .build(), + ]; + log::info!("Document filter list: {:?}", ssm_client_filters); + + let mut document_list: Vec = vec![]; + + if let Some(ref ssm_client) = client { + let mut paginator = ssm_client + .list_documents() + .set_filters(Some(ssm_client_filters)) + .into_paginator() + .send(); + + while let Some(result) = paginator.next().await { + if result.is_err() { + return Err(format!("SSM ListDocuments Failed: {}", result.unwrap_err())); + } + + let document_ids = Vec::from(result.unwrap().document_identifiers()); + + for document in document_ids { + document_list.push(document.name.unwrap()) + } + } + } + + log::info!("Found documents to delete {:?}", document_list); + + Ok(document_list) +}