update testing for #205

* bump to pop 17 and acct 6
* fix the acct-v6 problem in pytest-pop, bump to new version  (related https://gitlab.com/saltstack/pop/pytest-pop/-/merge_requests/4)
* make resource_group fixture allocate the resource group instead of just creating a string
* update resource group test since it is now a fixture
* on shutdown of the test session, remove the resource group which should cascade to remove all resources in the group
* move duplicated password fixture up to the common fixture config
* mark some tests as expensive and document testing

Signed-off-by: Tom Scanlan <tscanlan@vmware.com>

README.md

@@ -16,18 +16,6 @@ The azurerm idem provider can be installed via pip:
 pip install idem-azurerm
 ```
 
-## INSTALLATION FOR DEVELOPMENT
-1. Clone the `idem-azurerm` repository.
-2. Install requirements with pip:
-```
-pip install -r requirements.txt
-```
-3. Install `idem-azurerm` in "editable" mode:
-```
-pip install -e <path cloned repo>
-```
-You are now fully set up to begin developing additional functionality for this provider.
-
 ## CREDENTIALS
 This provider requires that a dictionary populated with valid Azure credentials be passed via
 [acct](https://gitlab.com/saltstack/pop/acct).
@@ -92,3 +80,49 @@ by running idem like so:
 ```
 (env) $ idem state mytest.sls
 ```
+
+## DEVELOPMENT
+
+### Installation For Development
+
+1. Clone the `idem-azurerm` repository.
+2. Install requirements with pip:
+
+```
+pip install -r requirements.txt
+```
+
+3. Install `idem-azurerm` in "editable" mode:
+
+```
+pip install -e <path cloned repo>
+```
+
+You are now fully set up to begin developing additional functionality for this provider.
+
+### Integration Tests
+
+Integration tests run against Azure using credentials as detailed [above](#CREDENTIALS). Keep in mind running
+integration tests incur real costs for your subscription. The Resource Group used for testing ought to be cleaned up at
+the end of the `pytest` run, but ***if the test process is abnormally interrupted clean up may not happen, and you'll
+need to manually remove any resources that remain***.
+
+#### IAM requirements
+
+To run tests you will need to create an App Registration, and under your subscription add a Role Assignment for it.
+Minimally, assign the role 'Contributor' to run the majority of tests. If you want to run all tests additional
+authorization is required, but currently undocumented.
+
+#### Run The Tests
+
+After creating credentials and exporting them as above, to run all but the expensive tests:
+
+```shell
+pytest
+```
+
+By default, tests marked as `@pytest.mark.expensive_test` will not be run, but to include them:
+
+```shell
+pytest --run-expensive
+```

requirements-test.txt

@@ -2,7 +2,7 @@ asynctest>=0.13.0
 mock>=4.0.2
 pytest>=6.0.1
 pytest-asyncio>=0.14.0
-pytest-pop>=6.3
+pytest-pop>=6.5.0
 pytest-subtests>=0.4.0
 pytest-ordering>=0.6
 sphinx

requirements.txt

@@ -1,5 +1,5 @@
-pop==13.2
-acct==2.3
+pop~=17.1
+acct==6
 idem==7.4
 pop-config==6.10
 takara==1.2

tests/conftest.py

@@ -1,6 +1,5 @@
 # -*- coding: utf-8 -*-
 
-# Import python libs
 import os
 import sys
 import logging

tests/integration/conftest.py

@@ -46,11 +46,17 @@ def tags():
 
 
 @pytest.fixture(scope="session")
-def resource_group():
-    yield "rg-idem-inttest-" + "".join(
+async def resource_group(hub, ctx, location):
+    name = "rg-idem-inttest-" + "".join(
         random.choice(string.ascii_lowercase + string.digits) for _ in range(20)
     )
 
+    await hub.states.azurerm.resource.group.present(ctx, name, location)
+    yield name
+
+    # Clean up
+    await hub.states.azurerm.resource.group.absent(ctx, name, location)
+
 
 @pytest.fixture(scope="session")
 def vm():
@@ -190,3 +196,10 @@ def acr():
     yield "acrideminttest" + "".join(
         random.choice(string.ascii_lowercase + string.digits) for _ in range(8)
     )
+
+
+@pytest.fixture(scope="session")
+def password():
+    yield "#PASS" + "".join(
+        random.choice(string.ascii_lowercase + string.digits) for _ in range(16)
+    ) + "!"

tests/integration/states/azurerm/compute/test_virtual_machine.py

@@ -3,13 +3,6 @@
 import string
 
 
-@pytest.fixture(scope="session")
-def password():
-    yield "#PASS" + "".join(
-        random.choice(string.ascii_lowercase + string.digits) for _ in range(16)
-    ) + "!"
-
-
 @pytest.mark.run(order=5)
 @pytest.mark.asyncio
 async def test_present(hub, ctx, vm, resource_group, vnet, subnet, password):

tests/integration/states/azurerm/network/test_bastion_host.py

@@ -11,6 +11,7 @@ def bastion_host():
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 @pytest.mark.slow
 async def test_present(hub, ctx, bastion_host, resource_group, vnet, public_ip_addr):
@@ -63,6 +64,7 @@ async def test_present(hub, ctx, bastion_host, resource_group, vnet, public_ip_a
 
 
 @pytest.mark.run(order=4, after="test_present", before="test_absent")
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 @pytest.mark.slow
 async def test_changes(
@@ -95,6 +97,7 @@ async def test_changes(
 
 
 @pytest.mark.run(order=-4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 @pytest.mark.slow
 async def test_absent(hub, ctx, bastion_host, resource_group):

tests/integration/states/azurerm/postgresql/test_configuration.py

@@ -2,6 +2,7 @@
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present_and_changes(hub, ctx, postgresql_server, resource_group):
     config_name = "log_retention_days"

tests/integration/states/azurerm/postgresql/test_database.py

@@ -11,6 +11,7 @@ def postgresql_db():
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present(hub, ctx, postgresql_db, postgresql_server, resource_group):
     expected = {
@@ -38,6 +39,7 @@ async def test_present(hub, ctx, postgresql_db, postgresql_server, resource_grou
 
 
 @pytest.mark.run(order=-4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_absent(hub, ctx, postgresql_db, postgresql_server, resource_group):
     expected = {

tests/integration/states/azurerm/postgresql/test_firewall_rule.py

@@ -11,6 +11,7 @@ def fw_rule():
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present(hub, ctx, fw_rule, postgresql_server, resource_group):
     start_addr = "10.0.0.0"
@@ -42,6 +43,7 @@ async def test_present(hub, ctx, fw_rule, postgresql_server, resource_group):
 
 
 @pytest.mark.run(order=4, after="test_present", before="test_absent")
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_changes(hub, ctx, fw_rule, postgresql_server, resource_group):
     start_addr = "10.0.0.0"
@@ -65,6 +67,7 @@ async def test_changes(hub, ctx, fw_rule, postgresql_server, resource_group):
 
 
 @pytest.mark.run(order=-4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_absent(hub, ctx, fw_rule, postgresql_server, resource_group):
     expected = {

tests/integration/states/azurerm/postgresql/test_server.py

@@ -3,14 +3,8 @@
 import string
 
 
-@pytest.fixture(scope="session")
-def password():
-    yield "#" + "".join(
-        random.choice(string.ascii_lowercase + string.digits) for _ in range(16)
-    ) + "!"
-
-
 @pytest.mark.run(order=3)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present(hub, ctx, postgresql_server, resource_group, location, password):
     login = "dbadmin"
@@ -68,6 +62,7 @@ async def test_present(hub, ctx, postgresql_server, resource_group, location, pa
 
 
 @pytest.mark.run(order=3, after="test_present", before="test_absent")
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_changes(hub, ctx, postgresql_server, resource_group, location, password):
     login = "dbadmin"
@@ -94,6 +89,7 @@ async def test_changes(hub, ctx, postgresql_server, resource_group, location, pa
 
 
 @pytest.mark.run(order=-3)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_absent(hub, ctx, postgresql_server, resource_group):
     expected = {

tests/integration/states/azurerm/postgresql/test_server_security_alert_policy.py

@@ -2,6 +2,7 @@
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present(hub, ctx, postgresql_server, resource_group):
     name = "Default"
@@ -22,6 +23,7 @@ async def test_present(hub, ctx, postgresql_server, resource_group):
 
 
 @pytest.mark.run(order=4, after="test_present")
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_changes(hub, ctx, postgresql_server, resource_group):
     name = "Default"

tests/integration/states/azurerm/postgresql/test_virtual_network_rule.py

@@ -11,6 +11,7 @@ def vnet_rule():
 
 
 @pytest.mark.run(order=4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_present(
     hub, ctx, vnet_rule, postgresql_server, resource_group, subnet, vnet,
@@ -48,6 +49,7 @@ async def test_present(
 
 
 @pytest.mark.run(order=4, after="test_present", before="test_absent")
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_changes(
     hub, ctx, vnet_rule, postgresql_server, resource_group, subnet, vnet,
@@ -81,6 +83,7 @@ async def test_changes(
 
 
 @pytest.mark.run(order=-4)
+@pytest.mark.expensive_test
 @pytest.mark.asyncio
 async def test_absent(hub, ctx, vnet_rule, postgresql_server, resource_group):
     expected = {

tests/integration/states/azurerm/redis/test_redis_operations.py

@@ -16,6 +16,7 @@ def sku():
 
 
 @pytest.mark.run(order=3)
+@pytest.mark.expensive_test
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_present(hub, ctx, resource_group, location, sku, redis_cache):
@@ -59,6 +60,7 @@ async def test_present(hub, ctx, resource_group, location, sku, redis_cache):
 
 
 @pytest.mark.run(order=3, after="test_present", before="test_absent")
+@pytest.mark.expensive_test
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_changes(hub, ctx, resource_group, location, sku, redis_cache):
@@ -81,6 +83,7 @@ async def test_changes(hub, ctx, resource_group, location, sku, redis_cache):
 
 
 @pytest.mark.run(order=-3)
+@pytest.mark.expensive_test
 @pytest.mark.slow
 @pytest.mark.asyncio
 async def test_absent(hub, ctx, resource_group, redis_cache):

tests/integration/states/azurerm/resource/test_group.py

@@ -5,21 +5,13 @@
 @pytest.mark.asyncio
 async def test_present(hub, ctx, resource_group, location):
     expected = {
-        "changes": {
-            "new": {
-                "location": location,
-                "name": resource_group,
-                "type": "Microsoft.Resources/resourceGroups",
-                "properties": {"provisioning_state": "Succeeded"},
-            },
-            "old": {},
-        },
-        "comment": f"Resource group {resource_group} has been created.",
         "name": resource_group,
         "result": True,
+        "comment": f"Resource group {resource_group} is already present.",
+        "changes": {},
     }
+
     ret = await hub.states.azurerm.resource.group.present(ctx, resource_group, location)
-    ret["changes"]["new"].pop("id")
     assert ret == expected