From f177581164a10017d962cc2c6225e02e6db7e6de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 3 Mar 2021 11:34:47 -0800 Subject: [PATCH 1/5] Add perf test for certificates --- .../tests/perfstress_tests/README.md | 48 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/get_certificate.py | 48 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/README.md create mode 100644 sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/__init__.py create mode 100644 sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py diff --git a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/README.md b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..8d5f88788239 --- /dev/null +++ b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/README.md @@ -0,0 +1,48 @@ +# azure-keyvault-certificates Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the +`dev_requirements` install. Start by creating a new virtual environment for your perf tests. This will need to be a +Python 3 environment, preferably >=3.7. + +### Setup for test resources + +The following environment variables will need to be set for the tests to access the live resources: + +``` +AZURE_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_CLIENT_SECRET= +AZURE_KEYVAULT_URL= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-keyvault-certificates> pip install -r dev_requirements.txt +(env) ~/azure-keyvault-certificates> pip install -e . +``` + +## Test commands + +Once `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the +current module for runnable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-keyvault-certificates> cd tests/perfstress_tests/ +(env) ~/azure-keyvault-certificates/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-keyvault-certificates/tests> perfstress GetCertificateTest +``` diff --git a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/__init__.py b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py new file mode 100644 index 000000000000..b0f2a8b0622c --- /dev/null +++ b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py @@ -0,0 +1,48 @@ +from azure_devtools.perfstress_tests import PerfStressTest +from azure.identity import ClientSecretCredential +from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.keyvault.certificates import CertificateClient, CertificatePolicy +from azure.keyvault.certificates.aio import CertificateClient as AsyncCertificateClient + + +class GetCertificateTest(PerfStressTest): + + def __init__(self, arguments): + super().__init__(arguments) + + # Auth configuration + tenant_id = self.get_from_env("AZURE_TENANT_ID") + client_id = self.get_from_env("AZURE_CLIENT_ID") + client_secret = self.get_from_env("AZURE_CLIENT_SECRET") + self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) + self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + + # Create clients + vault_url = self.get_from_env("AZURE_KEYVAULT_URL") + self.client = CertificateClient(vault_url, self.credential) + self.async_client = AsyncCertificateClient(vault_url, self.async_credential) + + async def global_setup(self): + """The global setup is run only once.""" + await super().global_setup() + await self.async_client.create_certificate("livekvtestperfcert", CertificatePolicy.get_default()) + + async def global_cleanup(self): + """The global cleanup is run only once.""" + await self.async_client.delete_certificate("livekvtestperfcert") + await self.async_client.purge_deleted_certificate("livekvtestperfcert") + await super().global_cleanup() + + async def close(self): + """This is run after cleanup.""" + await self.async_client.close() + await self.async_credential.close() + await super().close() + + def run_sync(self): + """The synchronous perf test.""" + self.client.get_certificate("livekvtestperfcert") + + async def run_async(self): + """The asynchronous perf test.""" + await self.async_client.get_certificate("livekvtestperfcert") From 21ba8f1f75889afadfa1032564c5d5ea22234fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 3 Mar 2021 13:32:24 -0800 Subject: [PATCH 2/5] Add perf test for keys --- .../tests/perfstress_tests/README.md | 48 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/get_key.py | 48 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/README.md create mode 100644 sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/__init__.py create mode 100644 sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py diff --git a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/README.md b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..0acd4ff798f5 --- /dev/null +++ b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/README.md @@ -0,0 +1,48 @@ +# azure-keyvault-keys Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the +`dev_requirements` install. Start by creating a new virtual environment for your perf tests. This will need to be a +Python 3 environment, preferably >=3.7. + +### Setup for test resources + +The following environment variables will need to be set for the tests to access the live resources: + +``` +AZURE_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_CLIENT_SECRET= +AZURE_KEYVAULT_URL= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-keyvault-keys> pip install -r dev_requirements.txt +(env) ~/azure-keyvault-keys> pip install -e . +``` + +## Test commands + +Once `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the +current module for runnable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-keyvault-keys> cd tests/perfstress_tests/ +(env) ~/azure-keyvault-keys/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-keyvault-keys/tests> perfstress GetKeyTest +``` diff --git a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/__init__.py b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py new file mode 100644 index 000000000000..ebf58d04b7e0 --- /dev/null +++ b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py @@ -0,0 +1,48 @@ +from azure_devtools.perfstress_tests import PerfStressTest +from azure.identity import ClientSecretCredential +from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.keyvault.keys import KeyClient +from azure.keyvault.keys.aio import KeyClient as AsyncKeyClient + + +class GetKeyTest(PerfStressTest): + + def __init__(self, arguments): + super().__init__(arguments) + + # Auth configuration + tenant_id = self.get_from_env("AZURE_TENANT_ID") + client_id = self.get_from_env("AZURE_CLIENT_ID") + client_secret = self.get_from_env("AZURE_CLIENT_SECRET") + self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) + self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + + # Create clients + vault_url = self.get_from_env("AZURE_KEYVAULT_URL") + self.client = KeyClient(vault_url, self.credential) + self.async_client = AsyncKeyClient(vault_url, self.async_credential) + + async def global_setup(self): + """The global setup is run only once.""" + await super().global_setup() + await self.async_client.create_rsa_key("livekvtestperfkey") + + async def global_cleanup(self): + """The global cleanup is run only once.""" + await self.async_client.delete_key("livekvtestperfkey") + await self.async_client.purge_deleted_key("livekvtestperfkey") + await super().global_cleanup() + + async def close(self): + """This is run after cleanup.""" + await self.async_client.close() + await self.async_credential.close() + await super().close() + + def run_sync(self): + """The synchronous perf test.""" + self.client.get_key("livekvtestperfkey") + + async def run_async(self): + """The asynchronous perf test.""" + await self.async_client.get_key("livekvtestperfkey") From 3cd3f6d895102171818e83332bb4e4fdcd04c88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Wed, 3 Mar 2021 13:40:21 -0800 Subject: [PATCH 3/5] Add perf test for secrets --- .../tests/perfstress_tests/README.md | 48 +++++++++++++++++++ .../tests/perfstress_tests/__init__.py | 0 .../tests/perfstress_tests/get_secret.py | 48 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/README.md create mode 100644 sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/__init__.py create mode 100644 sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py diff --git a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/README.md b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/README.md new file mode 100644 index 000000000000..a446609bd630 --- /dev/null +++ b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/README.md @@ -0,0 +1,48 @@ +# azure-keyvault-secrets Performance Tests + +In order to run the performance tests, the `azure-devtools` package must be installed. This is done as part of the +`dev_requirements` install. Start by creating a new virtual environment for your perf tests. This will need to be a +Python 3 environment, preferably >=3.7. + +### Setup for test resources + +The following environment variables will need to be set for the tests to access the live resources: + +``` +AZURE_TENANT_ID= +AZURE_CLIENT_ID= +AZURE_CLIENT_SECRET= +AZURE_KEYVAULT_URL= +``` + +### Setup for perf test runs + +```cmd +(env) ~/azure-keyvault-secrets> pip install -r dev_requirements.txt +(env) ~/azure-keyvault-secrets> pip install -e . +``` + +## Test commands + +Once `azure-devtools` is installed, you will have access to the `perfstress` command line tool, which will scan the +current module for runnable perf tests. Only a specific test can be run at a time (i.e. there is no "run all" feature). + +```cmd +(env) ~/azure-keyvault-secrets> cd tests/perfstress_tests/ +(env) ~/azure-keyvault-secrets/tests> perfstress +``` +Using the `perfstress` command alone will list the available perf tests found. + +### Common perf command line options +These options are available for all perf tests: +- `--duration=10` Number of seconds to run as many operations (the "run" function) as possible. Default is 10. +- `--iterations=1` Number of test iterations to run. Default is 1. +- `--parallel=1` Number of tests to run in parallel. Default is 1. +- `--warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. +- `--sync` Whether to run the tests in sync or async. Default is False (async). +- `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). + +## Example command +```cmd +(env) ~/azure-keyvault-secrets/tests> perfstress GetSecretTest +``` diff --git a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/__init__.py b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py new file mode 100644 index 000000000000..ec6f40bcc93b --- /dev/null +++ b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py @@ -0,0 +1,48 @@ +from azure_devtools.perfstress_tests import PerfStressTest +from azure.identity import ClientSecretCredential +from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.keyvault.secrets import SecretClient +from azure.keyvault.secrets.aio import SecretClient as AsyncSecretClient + + +class GetSecretTest(PerfStressTest): + + def __init__(self, arguments): + super().__init__(arguments) + + # Auth configuration + tenant_id = self.get_from_env("AZURE_TENANT_ID") + client_id = self.get_from_env("AZURE_CLIENT_ID") + client_secret = self.get_from_env("AZURE_CLIENT_SECRET") + self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) + self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + + # Create clients + vault_url = self.get_from_env("AZURE_KEYVAULT_URL") + self.client = SecretClient(vault_url, self.credential) + self.async_client = AsyncSecretClient(vault_url, self.async_credential) + + async def global_setup(self): + """The global setup is run only once.""" + await super().global_setup() + await self.async_client.set_secret("livekvtestperfsecret", "secret-value") + + async def global_cleanup(self): + """The global cleanup is run only once.""" + await self.async_client.delete_secret("livekvtestperfsecret") + await self.async_client.purge_deleted_secret("livekvtestperfsecret") + await super().global_cleanup() + + async def close(self): + """This is run after cleanup.""" + await self.async_client.close() + await self.async_credential.close() + await super().close() + + def run_sync(self): + """The synchronous perf test.""" + self.client.get_secret("livekvtestperfsecret") + + async def run_async(self): + """The asynchronous perf test.""" + await self.async_client.get_secret("livekvtestperfsecret") From 2c0d7f1026b46ad6a7aa132d846334b2ef191c97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 4 Mar 2021 15:58:06 -0800 Subject: [PATCH 4/5] Use EnvrionmentCredential --- .../tests/perfstress_tests/get_certificate.py | 11 ++++------- .../tests/perfstress_tests/get_key.py | 11 ++++------- .../tests/perfstress_tests/get_secret.py | 11 ++++------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py index b0f2a8b0622c..15faa1413ccc 100644 --- a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py +++ b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py @@ -1,6 +1,6 @@ from azure_devtools.perfstress_tests import PerfStressTest -from azure.identity import ClientSecretCredential -from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.identity import EnvironmentCredential +from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential from azure.keyvault.certificates import CertificateClient, CertificatePolicy from azure.keyvault.certificates.aio import CertificateClient as AsyncCertificateClient @@ -11,11 +11,8 @@ def __init__(self, arguments): super().__init__(arguments) # Auth configuration - tenant_id = self.get_from_env("AZURE_TENANT_ID") - client_id = self.get_from_env("AZURE_CLIENT_ID") - client_secret = self.get_from_env("AZURE_CLIENT_SECRET") - self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) - self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + self.credential = EnvironmentCredential() + self.async_credential = AsyncEnvironmentCredential() # Create clients vault_url = self.get_from_env("AZURE_KEYVAULT_URL") diff --git a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py index ebf58d04b7e0..d38a2014fcac 100644 --- a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py +++ b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py @@ -1,6 +1,6 @@ from azure_devtools.perfstress_tests import PerfStressTest -from azure.identity import ClientSecretCredential -from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.identity import EnvironmentCredential +from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential from azure.keyvault.keys import KeyClient from azure.keyvault.keys.aio import KeyClient as AsyncKeyClient @@ -11,11 +11,8 @@ def __init__(self, arguments): super().__init__(arguments) # Auth configuration - tenant_id = self.get_from_env("AZURE_TENANT_ID") - client_id = self.get_from_env("AZURE_CLIENT_ID") - client_secret = self.get_from_env("AZURE_CLIENT_SECRET") - self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) - self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + self.credential = EnvironmentCredential() + self.async_credential = AsyncEnvironmentCredential() # Create clients vault_url = self.get_from_env("AZURE_KEYVAULT_URL") diff --git a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py index ec6f40bcc93b..80898c907ca4 100644 --- a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py +++ b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py @@ -1,6 +1,6 @@ from azure_devtools.perfstress_tests import PerfStressTest -from azure.identity import ClientSecretCredential -from azure.identity.aio import ClientSecretCredential as AsyncClientSecretCredential +from azure.identity import EnvironmentCredential +from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential from azure.keyvault.secrets import SecretClient from azure.keyvault.secrets.aio import SecretClient as AsyncSecretClient @@ -11,11 +11,8 @@ def __init__(self, arguments): super().__init__(arguments) # Auth configuration - tenant_id = self.get_from_env("AZURE_TENANT_ID") - client_id = self.get_from_env("AZURE_CLIENT_ID") - client_secret = self.get_from_env("AZURE_CLIENT_SECRET") - self.credential = ClientSecretCredential(tenant_id, client_id, client_secret) - self.async_credential = AsyncClientSecretCredential(tenant_id, client_id, client_secret) + self.credential = EnvironmentCredential() + self.async_credential = AsyncEnvironmentCredential() # Create clients vault_url = self.get_from_env("AZURE_KEYVAULT_URL") From 5ba279e2c732b36d71731b0ce0286858d98d9bb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?McCoy=20Pati=C3=B1o?= Date: Thu, 4 Mar 2021 16:00:12 -0800 Subject: [PATCH 5/5] Add license headers --- .../tests/perfstress_tests/get_certificate.py | 5 +++++ .../azure-keyvault-keys/tests/perfstress_tests/get_key.py | 5 +++++ .../tests/perfstress_tests/get_secret.py | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py index 15faa1413ccc..c6afd0fe3376 100644 --- a/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py +++ b/sdk/keyvault/azure-keyvault-certificates/tests/perfstress_tests/get_certificate.py @@ -1,3 +1,8 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + from azure_devtools.perfstress_tests import PerfStressTest from azure.identity import EnvironmentCredential from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential diff --git a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py index d38a2014fcac..cf9e8fbc22d3 100644 --- a/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py +++ b/sdk/keyvault/azure-keyvault-keys/tests/perfstress_tests/get_key.py @@ -1,3 +1,8 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + from azure_devtools.perfstress_tests import PerfStressTest from azure.identity import EnvironmentCredential from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential diff --git a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py index 80898c907ca4..b40bb6c20841 100644 --- a/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py +++ b/sdk/keyvault/azure-keyvault-secrets/tests/perfstress_tests/get_secret.py @@ -1,3 +1,8 @@ +# ------------------------------------ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# ------------------------------------ + from azure_devtools.perfstress_tests import PerfStressTest from azure.identity import EnvironmentCredential from azure.identity.aio import EnvironmentCredential as AsyncEnvironmentCredential