From b44b5012931c97d6f9609500611fee26a7908596 Mon Sep 17 00:00:00 2001 From: Marcos Borges Date: Sun, 14 Nov 2021 14:48:33 +0000 Subject: [PATCH 1/5] Example of distribuited execution on AWS EC2 using Terraform --- .../aws/README.md | 92 +++++++++++++++++++ .../aws/data_subnet.tf | 6 ++ .../aws/main.tf | 32 +++++++ .../aws/output.tf | 24 +++++ .../aws/plan/basic.py | 20 ++++ .../aws/provisioner.tf | 3 + .../aws/variables.tf | 17 ++++ 7 files changed, 194 insertions(+) create mode 100644 examples/distribuited_execution_terraform/aws/README.md create mode 100644 examples/distribuited_execution_terraform/aws/data_subnet.tf create mode 100644 examples/distribuited_execution_terraform/aws/main.tf create mode 100644 examples/distribuited_execution_terraform/aws/output.tf create mode 100644 examples/distribuited_execution_terraform/aws/plan/basic.py create mode 100644 examples/distribuited_execution_terraform/aws/provisioner.tf create mode 100644 examples/distribuited_execution_terraform/aws/variables.tf diff --git a/examples/distribuited_execution_terraform/aws/README.md b/examples/distribuited_execution_terraform/aws/README.md new file mode 100644 index 0000000000..3b620cd802 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/README.md @@ -0,0 +1,92 @@ + +## 1. AWS Authentication + +```bash +export AWS_ACCESS_KEY_ID=AIAXXXXXXXXXXXXXXXXX +export AWS_SECRET_ACCESS_KEY=T9HyXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + +## 2. Configure your provisioning + +- Don't forget to provide the correct subnet name in the variable file +- Define location and file of your locust plan script +- Define the number of nodes to create + +**variables.tf** + +```bash +variable "node_size" { + description = "Size of total nodes" + default = 2 +} + +variable "loadtest_dir_source" { + default = "plan/" +} + +variable "locust_plan_filename" { + default = "basic.py" +} + +variable "subnet_name" { + default = "subnet-prd-a" + description = "Subnet name" +} + +``` + + + +--- + +## 3. Execute Terraform + +```bash +cd examples/distribuited_execution_terraform/aws +terraform init +terraform apply --auto-approve +``` + +--- + +## 4. Access UI + +Click on the link below to access the UI: + +Result exemple: + +```bash +Apply complete! Resources: 14 added, 0 changed, 0 destroyed. + +Outputs: + +dashboard_url = "http://3.237.255.123" +leader_private_ip = "10.17.5.119" +leader_public_ip = "3.237.255.123" +nodes_private_ip = [ + "10.17.5.167", + "10.17.5.39", +] +nodes_public_ip = [ + "3.235.45.218", + "100.24.124.0", +] +``` + +![locust-home](https://github.com/marcosborges/terraform-aws-loadtest-distribuited/raw/v0.4.0/assets/locust-home.png) + +--- + +## 5. Cleanup + +```bash +terraform destroy --auto-approve +``` + +--- + +## 6. More information + +- [Terraform aws-get-started >> install-terraform-on-linux](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started#install-terraform-on-linux) + +- [Terraform module aws loadtest distribuited](https://registry.terraform.io/modules/marcosborges/loadtest-distribuited/aws/latest) \ No newline at end of file diff --git a/examples/distribuited_execution_terraform/aws/data_subnet.tf b/examples/distribuited_execution_terraform/aws/data_subnet.tf new file mode 100644 index 0000000000..0c7a599c95 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/data_subnet.tf @@ -0,0 +1,6 @@ +data "aws_subnet" "current" { + filter { + name = "tag:Name" + values = [var.subnet_name] + } +} \ No newline at end of file diff --git a/examples/distribuited_execution_terraform/aws/main.tf b/examples/distribuited_execution_terraform/aws/main.tf new file mode 100644 index 0000000000..8ab7b1fbb7 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/main.tf @@ -0,0 +1,32 @@ +module "loadtest" { + + # https://registry.terraform.io/modules/marcosborges/loadtest-distribuited/aws/latest + source = "marcosborges/loadtest-distribuited/aws" + + name = "provision-name" + nodes_size = var.node_size + executor = "locust" + loadtest_dir_source = var.loadtest_dir_source + + # LEADER ENTRYPOINT + loadtest_entrypoint = <<-EOT + nohup locust \ + -f ${var.locust_plan_filename} \ + --web-port=8080 \ + --expect-workers=${var.node_size} \ + --master > locust-leader.out 2>&1 & + EOT + + # NODES ENTRYPOINT + node_custom_entrypoint = <<-EOT + nohup locust \ + -f ${var.locust_plan_filename} \ + --worker \ + --master-host={LEADER_IP} > locust-worker.out 2>&1 & + EOT + + subnet_id = data.aws_subnet.current.id + locust_plan_filename = var.locust_plan_filename + ssh_export_pem = false + +} \ No newline at end of file diff --git a/examples/distribuited_execution_terraform/aws/output.tf b/examples/distribuited_execution_terraform/aws/output.tf new file mode 100644 index 0000000000..5a42883397 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/output.tf @@ -0,0 +1,24 @@ +output "leader_public_ip" { + value = module.loadtest.leader_public_ip + description = "The public IP address of the leader server instance." +} + +output "leader_private_ip" { + value = module.loadtest.leader_private_ip + description = "The private IP address of the leader server instance." +} + +output "nodes_public_ip" { + value = module.loadtest.nodes_public_ip + description = "The public IP address of the nodes instances." +} + +output "nodes_private_ip" { + value = module.loadtest.nodes_private_ip + description = "The private IP address of the nodes instances." +} + +output "dashboard_url" { + value = "http://${coalesce(module.loadtest.leader_public_ip, module.loadtest.leader_private_ip)}" + description = "The URL of the Locust UI." +} \ No newline at end of file diff --git a/examples/distribuited_execution_terraform/aws/plan/basic.py b/examples/distribuited_execution_terraform/aws/plan/basic.py new file mode 100644 index 0000000000..1036abe365 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/plan/basic.py @@ -0,0 +1,20 @@ +import time +from locust import HttpUser, task, between + +class Quickstart(HttpUser): + wait_time = between(1, 5) + + @task + def google(self): + self.client.request_name = "google" + self.client.get("https://google.com/") + + @task + def microsoft(self): + self.client.request_name = "microsoft" + self.client.get("https://microsoft.com/") + + @task + def facebook(self): + self.client.request_name = "facebook" + self.client.get("https://facebook.com/") diff --git a/examples/distribuited_execution_terraform/aws/provisioner.tf b/examples/distribuited_execution_terraform/aws/provisioner.tf new file mode 100644 index 0000000000..521cb844cb --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/provisioner.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "us-east-1" +} \ No newline at end of file diff --git a/examples/distribuited_execution_terraform/aws/variables.tf b/examples/distribuited_execution_terraform/aws/variables.tf new file mode 100644 index 0000000000..bf816667f4 --- /dev/null +++ b/examples/distribuited_execution_terraform/aws/variables.tf @@ -0,0 +1,17 @@ +variable "node_size" { + description = "Size of total nodes" + default = 2 +} + +variable "loadtest_dir_source" { + default = "plan/" +} + +variable "locust_plan_filename" { + default = "basic.py" +} + +variable "subnet_name" { + default = "subnet-prd-a" + description = "Subnet name" +} \ No newline at end of file From e36107e23e231512c0c758c472c7af0297b59356 Mon Sep 17 00:00:00 2001 From: Marcos Borges Date: Sun, 14 Nov 2021 14:52:54 +0000 Subject: [PATCH 2/5] Fix new line --- examples/distribuited_execution_terraform/aws/README.md | 3 ++- examples/distribuited_execution_terraform/aws/data_subnet.tf | 2 +- examples/distribuited_execution_terraform/aws/main.tf | 2 +- examples/distribuited_execution_terraform/aws/output.tf | 2 +- examples/distribuited_execution_terraform/aws/provisioner.tf | 2 +- examples/distribuited_execution_terraform/aws/variables.tf | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/examples/distribuited_execution_terraform/aws/README.md b/examples/distribuited_execution_terraform/aws/README.md index 3b620cd802..690912b81e 100644 --- a/examples/distribuited_execution_terraform/aws/README.md +++ b/examples/distribuited_execution_terraform/aws/README.md @@ -89,4 +89,5 @@ terraform destroy --auto-approve - [Terraform aws-get-started >> install-terraform-on-linux](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started#install-terraform-on-linux) -- [Terraform module aws loadtest distribuited](https://registry.terraform.io/modules/marcosborges/loadtest-distribuited/aws/latest) \ No newline at end of file +- [Terraform module aws loadtest distribuited](https://registry.terraform.io/modules/marcosborges/loadtest-distribuited/aws/latest) + diff --git a/examples/distribuited_execution_terraform/aws/data_subnet.tf b/examples/distribuited_execution_terraform/aws/data_subnet.tf index 0c7a599c95..ca57d192f0 100644 --- a/examples/distribuited_execution_terraform/aws/data_subnet.tf +++ b/examples/distribuited_execution_terraform/aws/data_subnet.tf @@ -3,4 +3,4 @@ data "aws_subnet" "current" { name = "tag:Name" values = [var.subnet_name] } -} \ No newline at end of file +} diff --git a/examples/distribuited_execution_terraform/aws/main.tf b/examples/distribuited_execution_terraform/aws/main.tf index 8ab7b1fbb7..7c77885bf1 100644 --- a/examples/distribuited_execution_terraform/aws/main.tf +++ b/examples/distribuited_execution_terraform/aws/main.tf @@ -29,4 +29,4 @@ module "loadtest" { locust_plan_filename = var.locust_plan_filename ssh_export_pem = false -} \ No newline at end of file +} diff --git a/examples/distribuited_execution_terraform/aws/output.tf b/examples/distribuited_execution_terraform/aws/output.tf index 5a42883397..26965fdc47 100644 --- a/examples/distribuited_execution_terraform/aws/output.tf +++ b/examples/distribuited_execution_terraform/aws/output.tf @@ -21,4 +21,4 @@ output "nodes_private_ip" { output "dashboard_url" { value = "http://${coalesce(module.loadtest.leader_public_ip, module.loadtest.leader_private_ip)}" description = "The URL of the Locust UI." -} \ No newline at end of file +} diff --git a/examples/distribuited_execution_terraform/aws/provisioner.tf b/examples/distribuited_execution_terraform/aws/provisioner.tf index 521cb844cb..cee7b7e16b 100644 --- a/examples/distribuited_execution_terraform/aws/provisioner.tf +++ b/examples/distribuited_execution_terraform/aws/provisioner.tf @@ -1,3 +1,3 @@ provider "aws" { region = "us-east-1" -} \ No newline at end of file +} diff --git a/examples/distribuited_execution_terraform/aws/variables.tf b/examples/distribuited_execution_terraform/aws/variables.tf index bf816667f4..60ad06eb08 100644 --- a/examples/distribuited_execution_terraform/aws/variables.tf +++ b/examples/distribuited_execution_terraform/aws/variables.tf @@ -14,4 +14,4 @@ variable "locust_plan_filename" { variable "subnet_name" { default = "subnet-prd-a" description = "Subnet name" -} \ No newline at end of file +} From c2c4e47f5cdae6e683e87dcc8c7b536633755c5a Mon Sep 17 00:00:00 2001 From: Marcos Borges Date: Sun, 14 Nov 2021 17:30:54 +0000 Subject: [PATCH 3/5] fix with black formatter --- examples/distribuited_execution_terraform/aws/plan/basic.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/distribuited_execution_terraform/aws/plan/basic.py b/examples/distribuited_execution_terraform/aws/plan/basic.py index 1036abe365..99f7a11911 100644 --- a/examples/distribuited_execution_terraform/aws/plan/basic.py +++ b/examples/distribuited_execution_terraform/aws/plan/basic.py @@ -1,6 +1,7 @@ import time from locust import HttpUser, task, between + class Quickstart(HttpUser): wait_time = between(1, 5) From f99ed9967a0e67834c6c8031a0fcbe9f2a448c51 Mon Sep 17 00:00:00 2001 From: Marcos Borges Date: Sun, 14 Nov 2021 17:33:17 +0000 Subject: [PATCH 4/5] fix example dir renamed --- .../aws/README.md | 2 +- .../aws/data_subnet.tf | 0 .../{distribuited_execution_terraform => terraform}/aws/main.tf | 0 .../aws/output.tf | 0 .../aws/plan/basic.py | 0 .../aws/provisioner.tf | 0 .../aws/variables.tf | 0 7 files changed, 1 insertion(+), 1 deletion(-) rename examples/{distribuited_execution_terraform => terraform}/aws/README.md (97%) rename examples/{distribuited_execution_terraform => terraform}/aws/data_subnet.tf (100%) rename examples/{distribuited_execution_terraform => terraform}/aws/main.tf (100%) rename examples/{distribuited_execution_terraform => terraform}/aws/output.tf (100%) rename examples/{distribuited_execution_terraform => terraform}/aws/plan/basic.py (100%) rename examples/{distribuited_execution_terraform => terraform}/aws/provisioner.tf (100%) rename examples/{distribuited_execution_terraform => terraform}/aws/variables.tf (100%) diff --git a/examples/distribuited_execution_terraform/aws/README.md b/examples/terraform/aws/README.md similarity index 97% rename from examples/distribuited_execution_terraform/aws/README.md rename to examples/terraform/aws/README.md index 690912b81e..b01d740d4c 100644 --- a/examples/distribuited_execution_terraform/aws/README.md +++ b/examples/terraform/aws/README.md @@ -42,7 +42,7 @@ variable "subnet_name" { ## 3. Execute Terraform ```bash -cd examples/distribuited_execution_terraform/aws +cd examples/terraform/aws terraform init terraform apply --auto-approve ``` diff --git a/examples/distribuited_execution_terraform/aws/data_subnet.tf b/examples/terraform/aws/data_subnet.tf similarity index 100% rename from examples/distribuited_execution_terraform/aws/data_subnet.tf rename to examples/terraform/aws/data_subnet.tf diff --git a/examples/distribuited_execution_terraform/aws/main.tf b/examples/terraform/aws/main.tf similarity index 100% rename from examples/distribuited_execution_terraform/aws/main.tf rename to examples/terraform/aws/main.tf diff --git a/examples/distribuited_execution_terraform/aws/output.tf b/examples/terraform/aws/output.tf similarity index 100% rename from examples/distribuited_execution_terraform/aws/output.tf rename to examples/terraform/aws/output.tf diff --git a/examples/distribuited_execution_terraform/aws/plan/basic.py b/examples/terraform/aws/plan/basic.py similarity index 100% rename from examples/distribuited_execution_terraform/aws/plan/basic.py rename to examples/terraform/aws/plan/basic.py diff --git a/examples/distribuited_execution_terraform/aws/provisioner.tf b/examples/terraform/aws/provisioner.tf similarity index 100% rename from examples/distribuited_execution_terraform/aws/provisioner.tf rename to examples/terraform/aws/provisioner.tf diff --git a/examples/distribuited_execution_terraform/aws/variables.tf b/examples/terraform/aws/variables.tf similarity index 100% rename from examples/distribuited_execution_terraform/aws/variables.tf rename to examples/terraform/aws/variables.tf From 3b4f91af816f15dc8c4f0539c6a8f3eb93cfb42d Mon Sep 17 00:00:00 2001 From: Marcos Borges Date: Sun, 14 Nov 2021 17:46:44 +0000 Subject: [PATCH 5/5] Added documentation about distribuited cloud integration and IaC. --- docs/index.rst | 1 + docs/running-cloud-integration.rst | 103 +++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 docs/running-cloud-integration.rst diff --git a/docs/index.rst b/docs/index.rst index 9ba51b037f..61533d001d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,7 @@ Running your Locust tests configuration running-locust-distributed running-locust-docker + running-cloud-integration running-without-web-ui diff --git a/docs/running-cloud-integration.rst b/docs/running-cloud-integration.rst new file mode 100644 index 0000000000..e653532a98 --- /dev/null +++ b/docs/running-cloud-integration.rst @@ -0,0 +1,103 @@ +.. _running-cloud-integration: + +====================================================== +Running Locust Distributed with IaC on IaaS (AWS/EC2) +====================================================== + +1. AWS Authentication +===================== + +```bash +export AWS_ACCESS_KEY_ID=AIAXXXXXXXXXXXXXXXXX +export AWS_SECRET_ACCESS_KEY=T9HyXXXXXXXXXXXXXXXXXXXXXXXXXXXX +``` + +2. Configure your provisioning +============================== + +- Don't forget to provide the correct subnet name in the variable file +- Define location and file of your locust plan script +- Define the number of nodes to create + +**variables.tf** + +```bash +variable "node_size" { + description = "Size of total nodes" + default = 2 +} + +variable "loadtest_dir_source" { + default = "plan/" +} + +variable "locust_plan_filename" { + default = "basic.py" +} + +variable "subnet_name" { + default = "subnet-prd-a" + description = "Subnet name" +} + +``` + +--- + +3. Execute Terraform +======================= + +```bash +cd examples/terraform/aws +terraform init +terraform apply --auto-approve +``` + +--- + +4. Access UI +============ + + +Click on the link below to access the UI: + +Result exemple: + +```bash +Apply complete! Resources: 14 added, 0 changed, 0 destroyed. + +Outputs: + +dashboard_url = "http://3.237.255.123" +leader_private_ip = "10.17.5.119" +leader_public_ip = "3.237.255.123" +nodes_private_ip = [ + "10.17.5.167", + "10.17.5.39", +] +nodes_public_ip = [ + "3.235.45.218", + "100.24.124.0", +] +``` + +![locust-home](https://github.com/marcosborges/terraform-aws-loadtest-distribuited/raw/v0.4.0/assets/locust-home.png) + +--- + +5. Cleanup +========== + +```bash +terraform destroy --auto-approve +``` + +--- + +6. More information +=================== + +- [Terraform aws-get-started >> install-terraform-on-linux](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started#install-terraform-on-linux) + +- [Terraform module aws loadtest distribuited](https://registry.terraform.io/modules/marcosborges/loadtest-distribuited/aws/latest) +