From 4b96959fe0da021f3f51ee6be1c36bba8e7e8fce Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Wed, 6 Nov 2024 16:20:03 -0700 Subject: [PATCH 1/8] changes for opssw test --- .../constructs/ialirt_processing_construct.py | 66 ++++++++++--------- sds_data_manager/utils/stackbuilder.py | 8 +-- tests/test-data/ialirt_ec2/Dockerfile | 2 +- tests/test-data/ialirt_ec2/entrypoint.sh | 2 +- tests/test-data/ialirt_ec2/start_flask.sh | 13 +++- tests/test-data/ialirt_ec2/test_app.py | 7 +- 6 files changed, 55 insertions(+), 43 deletions(-) diff --git a/sds_data_manager/constructs/ialirt_processing_construct.py b/sds_data_manager/constructs/ialirt_processing_construct.py index a97ca3ab..d36e5bcb 100644 --- a/sds_data_manager/constructs/ialirt_processing_construct.py +++ b/sds_data_manager/constructs/ialirt_processing_construct.py @@ -7,7 +7,7 @@ https://aws.amazon.com/elasticloadbalancing/features/#Product_comparisons """ -from aws_cdk import CfnOutput, RemovalPolicy +from aws_cdk import CfnOutput, Duration, RemovalPolicy from aws_cdk import aws_autoscaling as autoscaling from aws_cdk import aws_ec2 as ec2 from aws_cdk import aws_ecs as ecs @@ -27,8 +27,7 @@ def __init__( construct_id: str, vpc: ec2.Vpc, processing_name: str, - ialirt_ports: list[int], - container_port: int, + ports: list[int], ialirt_bucket: s3.Bucket, secret_name: str, **kwargs, @@ -45,10 +44,8 @@ def __init__( VPC into which to put the resources that require networking. processing_name : str Name of the processing stack. - ialirt_ports : list[int] - List of ports to listen on for incoming traffic. - container_port : int - Port to be used by the container. + ports : list[int] + List of ports to listen on for incoming traffic and used by container. ialirt_bucket: s3.Bucket S3 bucket secret_name : str, @@ -59,8 +56,7 @@ def __init__( """ super().__init__(scope, construct_id, **kwargs) - self.ports = ialirt_ports - self.container_port = container_port + self.ports = ports self.vpc = vpc self.s3_bucket_name = ialirt_bucket.bucket_name self.secret_name = secret_name @@ -89,14 +85,14 @@ def create_ecs_security_group(self, processing_name): ) # Only allow traffic from the NLB security group - self.ecs_security_group.add_ingress_rule( - peer=ec2.Peer.security_group_id( - self.load_balancer_security_group.security_group_id - ), - connection=ec2.Port.tcp(self.container_port), - description=f"Allow inbound traffic from the NLB on " - f"TCP port {self.container_port}", - ) + for port in self.ports: + self.ecs_security_group.add_ingress_rule( + peer=ec2.Peer.security_group_id( + self.load_balancer_security_group.security_group_id + ), + connection=ec2.Port.tcp(port), + description=f"Allow inbound traffic from the NLB on TCP port {port}", + ) def create_load_balancer_security_group(self, processing_name): """Create and return a security group for load balancers.""" @@ -207,8 +203,8 @@ def add_compute_resources(self, processing_name): # Allowable values: # https://docs.aws.amazon.com/cdk/api/v2/docs/ # aws-cdk-lib.aws_ecs.TaskDefinition.html#cpu - memory_limit_mib=512, - cpu=256, + memory_limit_mib=1024, + cpu=512, logging=ecs.LogDrivers.aws_logs(stream_prefix=f"Ialirt{processing_name}"), environment={"S3_BUCKET": self.s3_bucket_name}, # Ensure the ECS task is running in privileged mode, @@ -216,15 +212,15 @@ def add_compute_resources(self, processing_name): privileged=True, ) - # Map ports to container - # NLB needs to know which port on the EC2 instances - # it should forward the traffic to - port_mapping = ecs.PortMapping( - container_port=self.container_port, - host_port=self.container_port, - protocol=ecs.Protocol.TCP, - ) - container.add_port_mappings(port_mapping) + # Maps the port on the ECS host + # corresponding container_port. + for port in self.ports: + port_mapping = ecs.PortMapping( + container_port=port, + host_port=port, + protocol=ecs.Protocol.TCP, + ) + container.add_port_mappings(port_mapping) # ECS Service is a configuration that # ensures application can run and maintain @@ -239,6 +235,7 @@ def add_compute_resources(self, processing_name): vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), + health_check_grace_period=Duration.seconds(300), ) def add_autoscaling(self, processing_name): @@ -250,7 +247,7 @@ def add_autoscaling(self, processing_name): self, f"AutoScalingGroup{processing_name}", instance_type=ec2.InstanceType.of( - ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.MICRO + ec2.InstanceClass.BURSTABLE3, ec2.InstanceSize.SMALL ), machine_image=ecs.EcsOptimizedImage.amazon_linux2(), vpc=self.vpc, @@ -259,6 +256,13 @@ def add_autoscaling(self, processing_name): auto_scaling_group.apply_removal_policy(RemovalPolicy.DESTROY) + # Attach the AmazonSSMManagedInstanceCore policy for SSM access + auto_scaling_group.role.add_managed_policy( + iam.ManagedPolicy.from_aws_managed_policy_name( + "AmazonSSMManagedInstanceCore" + ) + ) + # integrates ECS with EC2 Auto Scaling Groups # to manage the scaling and provisioning of the underlying # EC2 instances based on the requirements of ECS tasks @@ -301,8 +305,8 @@ def add_load_balancer(self, processing_name): # Register the ECS service as a target for the listener listener.add_targets( - f"Target{processing_name}{self.container_port}", - port=self.container_port, + f"Target{processing_name}{port}", + port=port, targets=[self.ecs_service], ) diff --git a/sds_data_manager/utils/stackbuilder.py b/sds_data_manager/utils/stackbuilder.py index 8addc071..18ba8446 100644 --- a/sds_data_manager/utils/stackbuilder.py +++ b/sds_data_manager/utils/stackbuilder.py @@ -265,18 +265,16 @@ def build_sds( ) # All traffic to I-ALiRT is directed to listed container ports - ialirt_ports = {"Primary": [8080, 8081], "Secondary": [80]} - container_ports = {"Primary": 8080, "Secondary": 80} + ports = {"Primary": [80, 8080]} # , "Secondary": [80]} ialirt_secret_name = "nexus-credentials" # noqa - for primary_or_secondary in ialirt_ports: + for primary_or_secondary in ports: ialirt_processing_construct.IalirtProcessing( scope=ialirt_stack, construct_id=f"IalirtProcessing{primary_or_secondary}", vpc=networking.vpc, processing_name=primary_or_secondary, - ialirt_ports=ialirt_ports[primary_or_secondary], - container_port=container_ports[primary_or_secondary], + ports=ports[primary_or_secondary], ialirt_bucket=ialirt_bucket.ialirt_bucket, secret_name=ialirt_secret_name, ) diff --git a/tests/test-data/ialirt_ec2/Dockerfile b/tests/test-data/ialirt_ec2/Dockerfile index 13df9092..0b47e05c 100644 --- a/tests/test-data/ialirt_ec2/Dockerfile +++ b/tests/test-data/ialirt_ec2/Dockerfile @@ -29,7 +29,7 @@ RUN chmod +x /app/mount_s3.sh /app/start_flask.sh /app/entrypoint.sh # Make port 8080 available to the world outside this container # Note: The port number is changed from 8080 to 80 for the secondary system. -EXPOSE 80 +EXPOSE 80 8080 # Set the AWS region ENV AWS_REGION us-west-2 diff --git a/tests/test-data/ialirt_ec2/entrypoint.sh b/tests/test-data/ialirt_ec2/entrypoint.sh index 3c1f1c0e..24df4f87 100644 --- a/tests/test-data/ialirt_ec2/entrypoint.sh +++ b/tests/test-data/ialirt_ec2/entrypoint.sh @@ -3,5 +3,5 @@ # Mount the S3 bucket /app/mount_s3.sh -# Start the Flask application +# Start the Flask application on multiple ports /app/start_flask.sh diff --git a/tests/test-data/ialirt_ec2/start_flask.sh b/tests/test-data/ialirt_ec2/start_flask.sh index 2bedb084..ac0b95d4 100644 --- a/tests/test-data/ialirt_ec2/start_flask.sh +++ b/tests/test-data/ialirt_ec2/start_flask.sh @@ -1,4 +1,13 @@ #!/bin/bash -# Start the Flask application -python test_app.py +# Define the ports on which the Flask app should run +PORTS=(80 8080) + +# Loop through the ports and start a Flask instance for each +for port in "${PORTS[@]}"; do + echo "Starting Flask app on port $port" + FLASK_PORT=$port python3 /app/test_app.py & +done + +# Wait for all Flask instances to complete +wait diff --git a/tests/test-data/ialirt_ec2/test_app.py b/tests/test-data/ialirt_ec2/test_app.py index d2ab9789..43f2d3c8 100644 --- a/tests/test-data/ialirt_ec2/test_app.py +++ b/tests/test-data/ialirt_ec2/test_app.py @@ -3,7 +3,7 @@ A simple Flask web application designed to be Dockerized and deployed on an EC2 instance. Intended for verifying the successful deployment and operation in an ECR and EC2 setup. The application listens on all interfaces (0.0.0.0) at -port 8080, allowing external access for testing. +configurable ports, allowing external access for testing. """ import os @@ -12,8 +12,9 @@ # Create a Flask application app = Flask(__name__) -# Note: The port number is changed from 8080 to 80 in the secondary Dockerfile. -port = 80 + +# Default port, can be overridden by environment variable +port = int(os.getenv("FLASK_PORT", "80")) # Decorator that tells Flask what URL From fa068db345ac033b80d2f10269c4603ee04e7396 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Thu, 7 Nov 2024 17:53:01 -0700 Subject: [PATCH 2/8] checkpoint --- .../constructs/ialirt_processing_construct.py | 16 ++++- sds_data_manager/utils/stackbuilder.py | 2 +- .../test_ialirt_processing_construct.py | 9 ++- tests/test-data/ialirt_ec2/Dockerfile | 2 +- tests/test-data/ialirt_ec2/start_flask.sh | 13 +---- tests/test-data/ialirt_ec2/test_app.py | 58 +++++++++++-------- 6 files changed, 56 insertions(+), 44 deletions(-) diff --git a/sds_data_manager/constructs/ialirt_processing_construct.py b/sds_data_manager/constructs/ialirt_processing_construct.py index d36e5bcb..3e3cc702 100644 --- a/sds_data_manager/constructs/ialirt_processing_construct.py +++ b/sds_data_manager/constructs/ialirt_processing_construct.py @@ -197,7 +197,7 @@ def add_compute_resources(self, processing_name): container = task_definition.add_container( f"IalirtContainer{processing_name}", image=ecs.ContainerImage.from_registry( - f"lasp-registry.colorado.edu/ialirt/ialirt-{processing_name.lower()}:latest", + f"lasp-registry.colorado.edu/ialirt/ialirt-{processing_name.lower()}:test", credentials=nexus_secret, ), # Allowable values: @@ -305,9 +305,19 @@ def add_load_balancer(self, processing_name): # Register the ECS service as a target for the listener listener.add_targets( - f"Target{processing_name}{port}", + f"TargetGroup{processing_name}{port}", port=port, - targets=[self.ecs_service], + targets=[ + self.ecs_service.load_balancer_target( + container_name=f"IalirtContainer{processing_name}", + container_port=port, + ) + ], + health_check=elbv2.HealthCheck( + enabled=True, + port=str(port), + protocol=elbv2.Protocol.TCP, + ), ) # This simply prints the DNS name of the diff --git a/sds_data_manager/utils/stackbuilder.py b/sds_data_manager/utils/stackbuilder.py index 18ba8446..05492c6b 100644 --- a/sds_data_manager/utils/stackbuilder.py +++ b/sds_data_manager/utils/stackbuilder.py @@ -265,7 +265,7 @@ def build_sds( ) # All traffic to I-ALiRT is directed to listed container ports - ports = {"Primary": [80, 8080]} # , "Secondary": [80]} + ports = {"Primary": [1234, 1235]} # , "Secondary": [80]} ialirt_secret_name = "nexus-credentials" # noqa for primary_or_secondary in ports: diff --git a/tests/infrastructure/test_ialirt_processing_construct.py b/tests/infrastructure/test_ialirt_processing_construct.py index 9677be2d..a98605b5 100644 --- a/tests/infrastructure/test_ialirt_processing_construct.py +++ b/tests/infrastructure/test_ialirt_processing_construct.py @@ -20,10 +20,9 @@ def get_nlb_dns(stack_name, port, container_name): @pytest.mark.xfail(reason="Will fail unless IALiRT stack is deployed.") def test_nlb_response(): """Test to ensure the NLB responds with HTTP 200 status.""" - ialirt_ports = {"Primary": [8080, 8081], "Secondary": [80]} - container_ports = {"Primary": 8080, "Secondary": 80} + ports = {"Primary": [1234, 1235], "Secondary": [1234, 1235]} - for stack_name, ports in ialirt_ports.items(): + for stack_name, ports in ports.items(): for port in ports: nlb_dns = get_nlb_dns(f"IalirtProcessing{stack_name}", port, stack_name) print(f"Testing URL: {nlb_dns}") @@ -33,11 +32,11 @@ def test_nlb_response(): response.status_code == 200 ), f"NLB did not return HTTP 200 on port {port} for {stack_name}" assert ( - response.text == f"Hello World from Port {container_ports[stack_name]}." + response.text == f"Hello World from Port {ports[stack_name]}." ), f"NLB did not return expected text on port {port} for {stack_name}" s3_response = requests.get( nlb_dns + "/list", timeout=10 ) # timeout in seconds assert ( - f"test_file{container_ports[stack_name]}.txt" in s3_response.text + f"test_file{ports[stack_name]}.txt" in s3_response.text ), f"NLB did not return expected file name on port {port} for {stack_name}" diff --git a/tests/test-data/ialirt_ec2/Dockerfile b/tests/test-data/ialirt_ec2/Dockerfile index 0b47e05c..1b3c3b47 100644 --- a/tests/test-data/ialirt_ec2/Dockerfile +++ b/tests/test-data/ialirt_ec2/Dockerfile @@ -29,7 +29,7 @@ RUN chmod +x /app/mount_s3.sh /app/start_flask.sh /app/entrypoint.sh # Make port 8080 available to the world outside this container # Note: The port number is changed from 8080 to 80 for the secondary system. -EXPOSE 80 8080 +EXPOSE 1234 1235 # Set the AWS region ENV AWS_REGION us-west-2 diff --git a/tests/test-data/ialirt_ec2/start_flask.sh b/tests/test-data/ialirt_ec2/start_flask.sh index ac0b95d4..2bedb084 100644 --- a/tests/test-data/ialirt_ec2/start_flask.sh +++ b/tests/test-data/ialirt_ec2/start_flask.sh @@ -1,13 +1,4 @@ #!/bin/bash -# Define the ports on which the Flask app should run -PORTS=(80 8080) - -# Loop through the ports and start a Flask instance for each -for port in "${PORTS[@]}"; do - echo "Starting Flask app on port $port" - FLASK_PORT=$port python3 /app/test_app.py & -done - -# Wait for all Flask instances to complete -wait +# Start the Flask application +python test_app.py diff --git a/tests/test-data/ialirt_ec2/test_app.py b/tests/test-data/ialirt_ec2/test_app.py index 43f2d3c8..2c5e10a6 100644 --- a/tests/test-data/ialirt_ec2/test_app.py +++ b/tests/test-data/ialirt_ec2/test_app.py @@ -3,36 +3,32 @@ A simple Flask web application designed to be Dockerized and deployed on an EC2 instance. Intended for verifying the successful deployment and operation in an ECR and EC2 setup. The application listens on all interfaces (0.0.0.0) at -configurable ports, allowing external access for testing. +multiple ports, allowing external access for testing. """ import os - from flask import Flask +from multiprocessing import Process -# Create a Flask application -app = Flask(__name__) - -# Default port, can be overridden by environment variable -port = int(os.getenv("FLASK_PORT", "80")) - +# Function to create a Flask application for a specific port +def create_app(port): + app = Flask(__name__) -# Decorator that tells Flask what URL -# should trigger the function that follows. -@app.route("/") -def hello(): - """Hello world function to test with.""" - return f"Hello World from Port {port}." + @app.route("/") + def hello(): + """Hello world function to test with.""" + return f"Hello World from Port {port}." + @app.route("/list") + def list_files(): + """List files in the mounted S3 bucket.""" + files = os.listdir("/mnt/s3/packets") + return "
".join(files) -@app.route("/list") -def list_files(): - """List files in the mounted S3 bucket.""" - files = os.listdir("/mnt/s3/packets") - return "
".join(files) + return app - -def create_and_save_file(): +# Function to create a test file for each port +def create_and_save_file(port): """Create and save file to S3 bucket.""" s3_mount_dir = "/mnt/s3/packets" @@ -49,7 +45,23 @@ def create_and_save_file(): print(f"File {file_name} created and saved to {file_path}.") +# Function to run the Flask app on a specified port +def run_app(port): + app = create_app(port) + create_and_save_file(port) + app.run(host="0.0.0.0", port=port) if __name__ == "__main__": - create_and_save_file() - app.run(host="0.0.0.0", port=port) + # Define the ports on which to run the Flask instances + ports = [1234, 1235] + + # Start a separate process for each Flask instance on each port + processes = [] + for port in ports: + process = Process(target=run_app, args=(port,)) + process.start() + processes.append(process) + + # Wait for all processes to complete + for process in processes: + process.join() From feb65a07bfc173781d004caa7deb73780185d1d2 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 8 Nov 2024 02:37:59 -0700 Subject: [PATCH 3/8] works --- tests/test-data/ialirt_ec2/test_app.py | 52 ++------------------------ 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/tests/test-data/ialirt_ec2/test_app.py b/tests/test-data/ialirt_ec2/test_app.py index 2c5e10a6..0ff52525 100644 --- a/tests/test-data/ialirt_ec2/test_app.py +++ b/tests/test-data/ialirt_ec2/test_app.py @@ -1,14 +1,5 @@ -"""A simple, dockerized, deployable Flask web application. - -A simple Flask web application designed to be Dockerized and deployed on an -EC2 instance. Intended for verifying the successful deployment and operation in -an ECR and EC2 setup. The application listens on all interfaces (0.0.0.0) at -multiple ports, allowing external access for testing. -""" - -import os from flask import Flask -from multiprocessing import Process +import multiprocessing # Function to create a Flask application for a specific port def create_app(port): @@ -16,52 +7,17 @@ def create_app(port): @app.route("/") def hello(): - """Hello world function to test with.""" - return f"Hello World from Port {port}." - - @app.route("/list") - def list_files(): - """List files in the mounted S3 bucket.""" - files = os.listdir("/mnt/s3/packets") - return "
".join(files) - - return app - -# Function to create a test file for each port -def create_and_save_file(port): - """Create and save file to S3 bucket.""" - s3_mount_dir = "/mnt/s3/packets" - - if not os.path.exists(s3_mount_dir): - os.makedirs(s3_mount_dir) + return f"Hello from Port {port}!", 200 - file_name = f"test_file{port}.txt" - file_content = "Hello, this is a test file." - - file_path = os.path.join(s3_mount_dir, file_name) - - with open(file_path, "w") as file: - file.write(file_content) - - print(f"File {file_name} created and saved to {file_path}.") - -# Function to run the Flask app on a specified port -def run_app(port): - app = create_app(port) - create_and_save_file(port) app.run(host="0.0.0.0", port=port) if __name__ == "__main__": # Define the ports on which to run the Flask instances ports = [1234, 1235] + processes = [multiprocessing.Process(target=create_app, args=(port,)) for port in ports] - # Start a separate process for each Flask instance on each port - processes = [] - for port in ports: - process = Process(target=run_app, args=(port,)) + for process in processes: process.start() - processes.append(process) - # Wait for all processes to complete for process in processes: process.join() From e78dbbcfc355c82a8af42c7c7ade841eed38d92a Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 8 Nov 2024 09:25:11 -0700 Subject: [PATCH 4/8] updates on docstrings --- .../constructs/ialirt_processing_construct.py | 13 +++-- sds_data_manager/utils/stackbuilder.py | 2 +- tests/test-data/ialirt_ec2/Dockerfile | 4 +- tests/test-data/ialirt_ec2/entrypoint.sh | 2 +- tests/test-data/ialirt_ec2/test_app.py | 52 +++++++++++++++++-- 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/sds_data_manager/constructs/ialirt_processing_construct.py b/sds_data_manager/constructs/ialirt_processing_construct.py index 3e3cc702..60befcc0 100644 --- a/sds_data_manager/constructs/ialirt_processing_construct.py +++ b/sds_data_manager/constructs/ialirt_processing_construct.py @@ -7,7 +7,7 @@ https://aws.amazon.com/elasticloadbalancing/features/#Product_comparisons """ -from aws_cdk import CfnOutput, Duration, RemovalPolicy +from aws_cdk import CfnOutput, RemovalPolicy from aws_cdk import aws_autoscaling as autoscaling from aws_cdk import aws_ec2 as ec2 from aws_cdk import aws_ecs as ecs @@ -212,8 +212,9 @@ def add_compute_resources(self, processing_name): privileged=True, ) - # Maps the port on the ECS host - # corresponding container_port. + # Map ports to container + # NLB needs to know which port on the EC2 instances + # it should forward the traffic to for port in self.ports: port_mapping = ecs.PortMapping( container_port=port, @@ -235,7 +236,6 @@ def add_compute_resources(self, processing_name): vpc_subnets=ec2.SubnetSelection( subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS ), - health_check_grace_period=Duration.seconds(300), ) def add_autoscaling(self, processing_name): @@ -305,14 +305,17 @@ def add_load_balancer(self, processing_name): # Register the ECS service as a target for the listener listener.add_targets( - f"TargetGroup{processing_name}{port}", + f"Target{processing_name}{port}", port=port, + # Specifies the container and port to route traffic to. targets=[ self.ecs_service.load_balancer_target( container_name=f"IalirtContainer{processing_name}", container_port=port, ) ], + # Configures health checks for the target group + # to ensure traffic is routed only to healthy ECS tasks. health_check=elbv2.HealthCheck( enabled=True, port=str(port), diff --git a/sds_data_manager/utils/stackbuilder.py b/sds_data_manager/utils/stackbuilder.py index 05492c6b..415abadd 100644 --- a/sds_data_manager/utils/stackbuilder.py +++ b/sds_data_manager/utils/stackbuilder.py @@ -265,7 +265,7 @@ def build_sds( ) # All traffic to I-ALiRT is directed to listed container ports - ports = {"Primary": [1234, 1235]} # , "Secondary": [80]} + ports = {"Primary": [1234, 1235], "Secondary": [1236]} ialirt_secret_name = "nexus-credentials" # noqa for primary_or_secondary in ports: diff --git a/tests/test-data/ialirt_ec2/Dockerfile b/tests/test-data/ialirt_ec2/Dockerfile index 1b3c3b47..15d064db 100644 --- a/tests/test-data/ialirt_ec2/Dockerfile +++ b/tests/test-data/ialirt_ec2/Dockerfile @@ -27,8 +27,8 @@ COPY entrypoint.sh /app/entrypoint.sh # Make the scripts executable RUN chmod +x /app/mount_s3.sh /app/start_flask.sh /app/entrypoint.sh -# Make port 8080 available to the world outside this container -# Note: The port number is changed from 8080 to 80 for the secondary system. +# Make ports available to the world outside this container +# Note: The port numbers are changed for the secondary system. EXPOSE 1234 1235 # Set the AWS region diff --git a/tests/test-data/ialirt_ec2/entrypoint.sh b/tests/test-data/ialirt_ec2/entrypoint.sh index 24df4f87..3c1f1c0e 100644 --- a/tests/test-data/ialirt_ec2/entrypoint.sh +++ b/tests/test-data/ialirt_ec2/entrypoint.sh @@ -3,5 +3,5 @@ # Mount the S3 bucket /app/mount_s3.sh -# Start the Flask application on multiple ports +# Start the Flask application /app/start_flask.sh diff --git a/tests/test-data/ialirt_ec2/test_app.py b/tests/test-data/ialirt_ec2/test_app.py index 0ff52525..4542d5cc 100644 --- a/tests/test-data/ialirt_ec2/test_app.py +++ b/tests/test-data/ialirt_ec2/test_app.py @@ -1,20 +1,62 @@ -from flask import Flask +"""A simple, dockerized, deployable Flask web application. + +A simple Flask web application designed to be Dockerized and deployed on an +EC2 instance. Intended for verifying the successful deployment and operation in +an ECR and EC2 setup. The application listens on all interfaces (0.0.0.0) at +defined ports, allowing external access for testing. +""" + import multiprocessing +import os + +from flask import Flask + -# Function to create a Flask application for a specific port def create_app(port): + """Create Flask application for a specific port.""" app = Flask(__name__) + # Decorator that tells Flask what URL + # should trigger the function that follows. @app.route("/") def hello(): - return f"Hello from Port {port}!", 200 + """Hello world function to test with.""" + return f"Hello World from Port {port}." + + @app.route("/list") + def list_files(): + """List files in the mounted S3 bucket.""" + files = os.listdir("/mnt/s3/packets") + return "
".join(files) app.run(host="0.0.0.0", port=port) + +def create_and_save_file(port): + """Create and save file to S3 bucket.""" + s3_mount_dir = "/mnt/s3/packets" + + if not os.path.exists(s3_mount_dir): + os.makedirs(s3_mount_dir) + + file_name = f"test_file{port}.txt" + file_content = "Hello, this is a test file." + + file_path = os.path.join(s3_mount_dir, file_name) + + with open(file_path, "w") as file: + file.write(file_content) + + print(f"File {file_name} created and saved to {file_path}.") + + if __name__ == "__main__": - # Define the ports on which to run the Flask instances ports = [1234, 1235] - processes = [multiprocessing.Process(target=create_app, args=(port,)) for port in ports] + for port in ports: + create_and_save_file(port) + processes = [ + multiprocessing.Process(target=create_app, args=(port,)) for port in ports + ] for process in processes: process.start() From 910a36c4d5458b3cec93debe410c256bd5f2c0ad Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 8 Nov 2024 09:31:17 -0700 Subject: [PATCH 5/8] fixed bug --- sds_data_manager/constructs/ialirt_processing_construct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sds_data_manager/constructs/ialirt_processing_construct.py b/sds_data_manager/constructs/ialirt_processing_construct.py index 60befcc0..69d676e2 100644 --- a/sds_data_manager/constructs/ialirt_processing_construct.py +++ b/sds_data_manager/constructs/ialirt_processing_construct.py @@ -197,7 +197,7 @@ def add_compute_resources(self, processing_name): container = task_definition.add_container( f"IalirtContainer{processing_name}", image=ecs.ContainerImage.from_registry( - f"lasp-registry.colorado.edu/ialirt/ialirt-{processing_name.lower()}:test", + f"lasp-registry.colorado.edu/ialirt/ialirt-{processing_name.lower()}:latest", credentials=nexus_secret, ), # Allowable values: From f1a56c5856b07d61d097d76ba90046358b011917 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 8 Nov 2024 09:49:57 -0700 Subject: [PATCH 6/8] update to test --- tests/infrastructure/test_ialirt_processing_construct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/infrastructure/test_ialirt_processing_construct.py b/tests/infrastructure/test_ialirt_processing_construct.py index a98605b5..d745255f 100644 --- a/tests/infrastructure/test_ialirt_processing_construct.py +++ b/tests/infrastructure/test_ialirt_processing_construct.py @@ -20,9 +20,9 @@ def get_nlb_dns(stack_name, port, container_name): @pytest.mark.xfail(reason="Will fail unless IALiRT stack is deployed.") def test_nlb_response(): """Test to ensure the NLB responds with HTTP 200 status.""" - ports = {"Primary": [1234, 1235], "Secondary": [1234, 1235]} + ialirt_ports = {"Primary": [1234, 1235], "Secondary": [1236]} - for stack_name, ports in ports.items(): + for stack_name, ports in ialirt_ports.items(): for port in ports: nlb_dns = get_nlb_dns(f"IalirtProcessing{stack_name}", port, stack_name) print(f"Testing URL: {nlb_dns}") From d84b6104db2b39cbbc4d02aa5c61a6b1682a57f8 Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Fri, 8 Nov 2024 10:22:29 -0700 Subject: [PATCH 7/8] minor text update --- .../test_ialirt_processing_construct.py | 23 ++++++++++--------- tests/test-data/ialirt_ec2/test_app.py | 2 +- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/tests/infrastructure/test_ialirt_processing_construct.py b/tests/infrastructure/test_ialirt_processing_construct.py index d745255f..0dce361b 100644 --- a/tests/infrastructure/test_ialirt_processing_construct.py +++ b/tests/infrastructure/test_ialirt_processing_construct.py @@ -8,11 +8,11 @@ def get_nlb_dns(stack_name, port, container_name): """Retrieve DNS for the NLB from CloudFormation.""" client = boto3.client("cloudformation") - response = client.describe_constructs(StackName=stack_name) + response = client.describe_stacks(StackName=stack_name) output_key = f"LoadBalancerDNS{container_name}{port}" outputs = response["Stacks"][0]["Outputs"] for output in outputs: - if output["OutputKey"] == output_key: + if output_key in output["OutputKey"]: return output["OutputValue"] raise ValueError(f"DNS output not found for port {port} in stack.") @@ -20,23 +20,24 @@ def get_nlb_dns(stack_name, port, container_name): @pytest.mark.xfail(reason="Will fail unless IALiRT stack is deployed.") def test_nlb_response(): """Test to ensure the NLB responds with HTTP 200 status.""" - ialirt_ports = {"Primary": [1234, 1235], "Secondary": [1236]} + ialirt_ports = {"Primary": [1235, 1234], "Secondary": [1236]} - for stack_name, ports in ialirt_ports.items(): + for container_name, ports in ialirt_ports.items(): for port in ports: - nlb_dns = get_nlb_dns(f"IalirtProcessing{stack_name}", port, stack_name) + nlb_dns = get_nlb_dns("IalirtStack", port, container_name) print(f"Testing URL: {nlb_dns}") # Specify a timeout for the request response = requests.get(nlb_dns, timeout=10) # timeout in seconds assert ( response.status_code == 200 - ), f"NLB did not return HTTP 200 on port {port} for {stack_name}" + ), f"NLB did not return HTTP 200 on port {port} for {container_name}" assert ( - response.text == f"Hello World from Port {ports[stack_name]}." - ), f"NLB did not return expected text on port {port} for {stack_name}" + response.text == f"Hello from Port {port}!" + ), f"NLB did not return expected text on port {port} for {container_name}" s3_response = requests.get( nlb_dns + "/list", timeout=10 ) # timeout in seconds - assert ( - f"test_file{ports[stack_name]}.txt" in s3_response.text - ), f"NLB did not return expected file name on port {port} for {stack_name}" + assert f"test_file{port}.txt" in s3_response.text, ( + f"NLB did not return expected file name on port {port} " + f"for {container_name}" + ) diff --git a/tests/test-data/ialirt_ec2/test_app.py b/tests/test-data/ialirt_ec2/test_app.py index 4542d5cc..416a277d 100644 --- a/tests/test-data/ialirt_ec2/test_app.py +++ b/tests/test-data/ialirt_ec2/test_app.py @@ -21,7 +21,7 @@ def create_app(port): @app.route("/") def hello(): """Hello world function to test with.""" - return f"Hello World from Port {port}." + return f"Hello from Port {port}!" @app.route("/list") def list_files(): From a698a901243ad2391dc6d275090827f7b2f060aa Mon Sep 17 00:00:00 2001 From: Laura Sandoval Date: Tue, 12 Nov 2024 09:53:12 -0700 Subject: [PATCH 8/8] remove print statement --- tests/infrastructure/test_ialirt_processing_construct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/infrastructure/test_ialirt_processing_construct.py b/tests/infrastructure/test_ialirt_processing_construct.py index 0dce361b..cc1f23fc 100644 --- a/tests/infrastructure/test_ialirt_processing_construct.py +++ b/tests/infrastructure/test_ialirt_processing_construct.py @@ -25,7 +25,6 @@ def test_nlb_response(): for container_name, ports in ialirt_ports.items(): for port in ports: nlb_dns = get_nlb_dns("IalirtStack", port, container_name) - print(f"Testing URL: {nlb_dns}") # Specify a timeout for the request response = requests.get(nlb_dns, timeout=10) # timeout in seconds assert (