running parallel tests for ec2 (#16)

.github/workflows/main.yml

@@ -77,9 +77,18 @@ jobs:
         ls -la terraform-provider-aws/test-bin
 
     - name: Run ${{ matrix.service }} Tests
+      if: ${{ matrix.service != 'ec2' }}
       run: |
         python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v --ls-start --ls-image ${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }}
 
+    - name: Run ${{ matrix.service }} Tests
+      if: ${{ matrix.service == 'ec2' }}
+      run: |
+        IMAGE_TAG=${{ github.event.inputs.localstack-image || 'localstack/localstack:latest' }} docker-compose up -d
+        sleep 20
+        python -m pytest --junitxml=target/reports/pytest.xml terraform-provider-aws/internal/service/${{ matrix.service }} -s -v -n 2
+
+
     - name: Publish ${{ matrix.service }} Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()

.gitignore

@@ -5,4 +5,5 @@
 __pycache__
 target
 **/*.test
-report.xml
+report.xml
+volume

Makefile

@@ -23,4 +23,4 @@ install:
 	$(VENV_RUN); $(PIP_CMD) install -r requirements.txt
 
 format:
-	$(VENV_RUN); python -m isort .; python -m black .
+	$(VENV_RUN); python -m isort .; python -m black .

README.md

@@ -5,21 +5,23 @@ This is a test runner for localstack and terraform. It will run a test cases fro
 Purpose of this project is to externalize the test cases from the localstack repo and run them against localstack to gather parity metrics.
 
 ## Installation
-1. Clone the repository with submodules`
+1. Clone the repository with submodules
+   - `git clone git@github.com:localstack/localstack-terraform-test.git --recurse-submodules`
+   - Make sure you have the latest version of the submodules after switching to a different branch using `git submodule update --init --recursive`
 2. Run `make venv` to create a virtual environment
 3. Run `make install` to install the dependencies
 
 ## How to run?
 1. Run `python -m terraform_pytest.main patch` to apply the patch to the terraform provider aws
 2. Run `python -m terraform_pytest.main build -s s3` to build testing binary for the golang module
-3Now you are ready to use `python -m pytest` commands to list and run test cases from golang
+3. Now you are ready to use `python -m pytest` commands to list and run test cases from golang
 
 ## How to run test cases?
 - To list down all the test case from a specific service, run `python -m pytest terraform-provider-aws/internal/service/<service> --collect-only -q`
 - To run a specific test case, run `python -m pytest terraform-provider-aws/internal/service/<service>/<test-file> -k <test-case-name> --ls-start` or `python -m pytest terraform-provider-aws/internal/service/<service>/<test-file>::<test-case-name> --ls-start`
 - Additional environment variables can be added by appending it in the start of the command, i.e. `AWS_ALTERNATE_REGION='us-west-2' python -m pytest terraform-provider-aws/internal/service/<service>/<test-file>::<test-case-name> --ls-start`
 
-## Default environment variables
+## Default environment variables for Terraform Tests
 - **TF_ACC**: `1`
 - **AWS_ACCESS_KEY_ID**: `test`
 - **AWS_SECRET_ACCESS_KEY**: `test`
@@ -30,6 +32,11 @@ Purpose of this project is to externalize the test cases from the localstack rep
 - **AWS_ALTERNATE_REGION**: `us-east-2`
 - **AWS_THIRD_REGION**: `eu-west-1`
 
+## Environment variables for Localstack
+- **DEBUG**: `1`
+- **PROVIDER_OVERRIDE_S3**: `asf`
+- **FAIL_FAST**: `1`
+
 ## Options
 - `--ls-start`: Start localstack instance before running the test cases
 - `--ls-image`: Specify the localstack image to use, default is `localstack/localstack:latest`

conftest.py

@@ -85,31 +85,6 @@ def reportinfo(self):
         return self.path, 0, f"Test Case: {self.name}"
 
 
-class ReprCrash:
-    def __init__(self, message):
-        self.message = message
-
-
-class LongRepr:
-    def __init__(self, message, reason):
-        self.reprcrash = ReprCrash(message)
-        self.reason = reason
-
-    def __str__(self):
-        return self.reason
-
-
-@pytest.hookimpl(tryfirst=True, hookwrapper=True)
-def pytest_runtest_makereport(item, call):
-    outcome = yield
-    report = outcome.get_result()
-    if report.failed:
-        splits = report.longrepr.split("\n", 1)
-        longrepr = LongRepr(splits[0], splits[1])
-        delattr(report, "longrepr")
-        setattr(report, "longrepr", longrepr)
-
-
 class GoException(Exception):
     def __init__(self, returncode, stderr):
         self.returncode = returncode
@@ -123,7 +98,7 @@ def _docker_service_health(client):
 
 
 def _start_docker_container(client, config, localstack_image):
-    env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf"]
+    env_vars = ["DEBUG=1", "PROVIDER_OVERRIDE_S3=asf", "FAIL_FAST=1"]
     port_mappings = {
         "53/tcp": ("127.0.0.1", 53),
         "53/udp": ("127.0.0.1", 53),

docker-compose.yml

@@ -0,0 +1,18 @@
+version: "3.8"
+
+services:
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME-localstack_main}"
+    image: ${IMAGE_TAG-localstack/localstack:latest}
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+      - "127.0.0.1:4510-4559:4510-4559"  # external services port range
+    environment:
+      - DEBUG=1
+      - PROVIDER_OVERRIDE_S3=asf
+      - FAIL_FAST=1
+      - LAMBDA_EXECUTOR=${LAMBDA_EXECUTOR-}
+      - DOCKER_HOST=unix:///var/run/docker.sock
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"

pyproject.toml

@@ -2,16 +2,17 @@
 
 [tool.black]
 line_length = 100
-include = '(terraform_pytest\/.*\.py$|tests\/.*\.py$)'
+include = '(^terraform_pytest\/.*\.py$|^conftest.py$)'
 #extend_exclude = '()'
 
 [tool.isort]
 profile = 'black'
+skip_glob = ["terraform-provider-aws"]
 #extend_skip = []
 line_length = 100
 
 [tool.pytest.ini_options]
 testpaths = [
     "terraform-provider-aws/internal/service/",
 ]
-#confcutdir = "tests/conftest.py"
+filterwarnings =["ignore::DeprecationWarning"]

requirements.txt

@@ -3,4 +3,5 @@ pytest==7.2.0
 docker==6.0.1
 requests==2.28.2
 black>=22.1
-isort>=5.10
+isort>=5.10
+pytest-xdist>=3.1.0

terraform_pytest/utils.py

@@ -143,6 +143,16 @@ def build_test_bin(service, tf_root_path):
     if exists(_test_bin_abs_path):
         return
 
+    cmd = ["go", "mod", "tidy"]
+    return_code, stdout = execute_command(cmd, cwd=tf_root_path)
+    if return_code != 0:
+        raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}")
+
+    cmd = ["go", "mod", "vendor"]
+    return_code, stdout = execute_command(cmd, cwd=tf_root_path)
+    if return_code != 0:
+        raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}")
+
     cmd = [
         "go",
         "test",
@@ -153,7 +163,7 @@ def build_test_bin(service, tf_root_path):
     ]
     return_code, stdout = execute_command(cmd, cwd=tf_root_path)
     if return_code != 0:
-        raise Exception(f"Error while building test binary for {service}")
+        raise Exception(f"Error while building test binary for {service}\ntraceback: {stdout}")
 
     if exists(_test_bin_abs_path):
         chmod(_test_bin_abs_path, 0o755)