Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

E2E tests: Fix shared service and performance tests #1860

Merged
merged 15 commits into from
May 24, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,11 @@ shared_service_bundle = $(MAKE) bundle-build DIR=./templates/shared_services/$(1
&& $(MAKE) bundle-publish DIR=./templates/shared_services/$(1)/ \
&& $(MAKE) bundle-register DIR="./templates/shared_services/$(1)" BUNDLE_TYPE=shared_service

user_resource_bundle = $(MAKE) bundle-build DIR=./templates/workspace_services/$(1)/user_resources/$(2)/ \
&& $(MAKE) bundle-publish DIR=./templates/workspace_services/$(1)/user_resources/$(2) \
&& $(MAKE) bundle-register DIR="./templates/workspace_services/$(1)/user_resources/$(2)" BUNDLE_TYPE=user_resource WORKSPACE_SERVICE_NAME=tre-service-$(1)


deploy-shared-service:
@# NOTE: ACR_NAME below comes from the env files, so needs the double '$$'. Others are set on command execution and don't
$(call target_title, "Deploying ${DIR} shared service") \
Expand Down Expand Up @@ -352,7 +357,8 @@ prepare-for-e2e:
&& $(call workspace_service_bundle,gitea) \
&& $(call workspace_service_bundle,innereye) \
&& $(call shared_service_bundle,sonatype-nexus) \
&& $(call shared_service_bundle,gitea)
&& $(call shared_service_bundle,gitea) \
&& $(call user_resource_bundle,guacamole,guacamole-dev-vm)

test-e2e-smoke:
$(call target_title, "Running E2E smoke tests") && \
Expand Down
48 changes: 33 additions & 15 deletions e2e_tests/test_performance.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
import asyncio
import pytest

import config
from helpers import disable_and_delete_resource, post_resource
from helpers import disable_and_delete_resource, get_workspace_owner_token, post_resource
from resources import strings

pytestmark = pytest.mark.asyncio


@pytest.mark.performance
@pytest.mark.timeout(3000)
async def test_parallel_resource_creations(admin_token, workspace_owner_token, verify) -> None:
async def test_parallel_resource_creations(admin_token, verify) -> None:
"""Creates N workspaces in parallel, and creates a workspace service in each, in parallel"""

number_workspaces = 2
Expand All @@ -28,23 +27,24 @@ async def test_parallel_resource_creations(admin_token, workspace_owner_token, v
}
}

task = asyncio.create_task(post_resource(payload, strings.API_WORKSPACES, 'workspace', workspace_owner_token, admin_token, verify))
task = asyncio.create_task(post_resource(payload=payload, endpoint=strings.API_WORKSPACES, access_token=admin_token, verify=verify))
tasks.append(task)

resource_paths = await asyncio.gather(*tasks)

# Now disable + delete them all in parallel
tasks = []
for ws, _ in resource_paths:
task = asyncio.create_task(disable_and_delete_resource(f'/api{ws}', 'workspace', workspace_owner_token, admin_token, verify))
for workspace_path, _ in resource_paths:
task = asyncio.create_task(disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify))
tasks.append(task)

await asyncio.gather(*tasks)


@pytest.mark.skip
@pytest.mark.performance
@pytest.mark.timeout(3000)
async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_token, workspace_owner_token, verify) -> None:
async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_token, verify) -> None:
"""Optionally creates a workspace and workspace service,
then creates N number of VMs in parallel, patches each, and deletes them"""

Expand All @@ -53,7 +53,9 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_toke

# To avoid creating + deleting a workspace + service in this test, set the vars for existing ones in ./templates/core/.env
# PERF_TEST_WORKSPACE_ID | PERF_TEST_WORKSPACE_SERVICE_ID
if config.PERF_TEST_WORKSPACE_ID == "":
workspace_id = config.PERF_TEST_WORKSPACE_ID

if workspace_id == "":
# create the workspace to use
payload = {
"templateName": "tre-workspace-base",
Expand All @@ -65,10 +67,12 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_toke
}
}

workspace_path, _ = await post_resource(payload, strings.API_WORKSPACES, 'workspace', workspace_owner_token, admin_token, verify)
workspace_path, workspace_id = await post_resource(payload, strings.API_WORKSPACES, admin_token, verify)
else:
workspace_path = f"/workspaces/{config.PERF_TEST_WORKSPACE_ID}"

workspace_owner_token = await get_workspace_owner_token(admin_token=admin_token, workspace_id=workspace_id, verify=verify)

if config.PERF_TEST_WORKSPACE_SERVICE_ID == "":
# create a guac service
service_payload = {
Expand All @@ -81,7 +85,11 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_toke
}
}

workspace_service_path, _ = await post_resource(service_payload, f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}', 'workspace_service', workspace_owner_token, None, verify)
workspace_service_path, _ = await post_resource(
payload=service_payload,
endpoint=f'/api{workspace_path}/{strings.API_WORKSPACE_SERVICES}',
access_token=workspace_owner_token,
verify=verify)
else:
workspace_service_path = f"{workspace_path}/{strings.API_WORKSPACE_SERVICES}/{config.PERF_TEST_WORKSPACE_SERVICE_ID}"

Expand All @@ -97,7 +105,11 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_toke

tasks = []
for i in range(number_vms):
task = asyncio.create_task(post_resource(user_resource_payload, f'/api{workspace_service_path}/{strings.API_USER_RESOURCES}', 'user_resource', workspace_owner_token, None, verify))
task = asyncio.create_task(post_resource(
payload=user_resource_payload,
endpoint=f'/api{workspace_service_path}/{strings.API_USER_RESOURCES}',
access_token=workspace_owner_token,
verify=verify))
tasks.append(task)

resource_paths = await asyncio.gather(*tasks)
Expand All @@ -113,17 +125,23 @@ async def test_bulk_updates_to_ensure_each_resource_updated_in_series(admin_toke
"display_name": f'Perf test VM update {i}',
}
}
await post_resource(patch_payload, f'/api{resource_path}', 'user_resource', workspace_owner_token, None, verify, method="PATCH", wait=False)
await post_resource(
payload=patch_payload,
endpoint=f'/api{resource_path}',
access_token=workspace_owner_token,
verify=verify,
method="PATCH",
wait=False)

# clear up all the VMs in parallel
# NOTE: Due to bug https://github.com/microsoft/AzureTRE/issues/1163 - this VM delete step currently fails
task = asyncio.create_task(disable_and_delete_resource(f'/api{resource_path}', 'user_resource', workspace_owner_token, None, verify))
task = asyncio.create_task(disable_and_delete_resource(f'/api{resource_path}', workspace_owner_token, verify))
tasks.append(task)

await asyncio.gather(*tasks)

# clear up workspace + service (if we created them)
if config.PERF_TEST_WORKSPACE_SERVICE_ID == "":
await disable_and_delete_resource(f'/api{workspace_service_path}', 'workspace_service', workspace_owner_token, None, verify)
await disable_and_delete_resource(f'/api{workspace_service_path}', workspace_owner_token, verify)
if config.PERF_TEST_WORKSPACE_ID == "":
await disable_and_delete_resource(f'/api{workspace_path}', 'workspace', workspace_owner_token, admin_token, verify)
await disable_and_delete_resource(f'/api{workspace_path}', admin_token, verify)
24 changes: 10 additions & 14 deletions e2e_tests/test_shared_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,10 @@ async def test_patch_firewall(admin_token, verify):
shared_service_path = f'/shared-services/{shared_service_firewall["id"]}'

await post_resource(
patch_payload,
f"/api{shared_service_path}",
"shared_service",
admin_token,
None,
verify,
payload=patch_payload,
endpoint=f"/api{shared_service_path}",
access_token=admin_token,
verify=verify,
method="PATCH",
)

Expand All @@ -109,7 +107,7 @@ async def test_create_shared_service(template_name, admin_token, verify) -> None
f"Shared service {template_name} already exists (id {id}), deleting it first..."
)
await disable_and_delete_resource(
f"/api/shared-services/{id}", "shared_service", admin_token, None, verify
f"/api/shared-services/{id}", admin_token, verify
)

post_payload = {
Expand All @@ -121,14 +119,12 @@ async def test_create_shared_service(template_name, admin_token, verify) -> None
}

shared_service_path, _ = await post_resource(
post_payload,
"/api/shared-services",
"shared_service",
admin_token,
None,
verify,
payload=post_payload,
endpoint="/api/shared-services",
access_token=admin_token,
verify=verify,
)

await disable_and_delete_resource(
f"/api{shared_service_path}", "shared_service", admin_token, None, verify
f"/api{shared_service_path}", admin_token, verify
)
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
name: tre-service-dev-vm
version: 0.3.0
version: 0.3.1
description: "An Azure TRE User Resource Template for a Dev VM"
registry: azuretre
dockerfile: Dockerfile.tmpl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"stepId": "6d2d7eb7-984e-4330-bd3c-c7ec98658402",
"stepTitle": "Update the firewall the first time",
"resourceTemplateName": "tre-shared-service-firewall",
"resourceType": "shared_service",
"resourceType": "shared-service",
"resourceAction": "upgrade",
"properties": [
{
Expand All @@ -53,7 +53,7 @@
"stepId": "2fe8a6a7-2c27-4c49-8773-127df8a48b4e",
"stepTitle": "Update the firewall the second time",
"resourceTemplateName": "tre-shared-service-firewall",
"resourceType": "shared_service",
"resourceType": "shared-service",
"resourceAction": "upgrade",
"properties": [
{
Expand Down
2 changes: 1 addition & 1 deletion templates/workspaces/base/porter.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
name: tre-workspace-base
version: 0.3.0
version: 0.3.1
description: "A base Azure TRE workspace"
registry: azuretre

Expand Down