From e5967e1f4e9ed01dacf9f11bf2bb6e7c4c824edd Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Sun, 16 Jan 2022 17:09:03 -0500 Subject: [PATCH 1/3] Move useful integration utils to main library Moving the Agent and Client pytest fixtures to the main library will allow use in external Agent development, such as socs. --- ocs/testing.py | 126 ++++++++++++++++++ .../test_aggregator_agent_integration.py | 5 +- .../test_fake_data_agent_integration.py | 7 +- .../test_host_master_agent_integration.py | 5 +- .../test_influxdb_publisher_integration.py | 5 +- .../test_registry_agent_integration.py | 5 +- tests/integration/util.py | 117 +--------------- 7 files changed, 152 insertions(+), 118 deletions(-) create mode 100644 ocs/testing.py diff --git a/ocs/testing.py b/ocs/testing.py new file mode 100644 index 00000000..649ed8ac --- /dev/null +++ b/ocs/testing.py @@ -0,0 +1,126 @@ +import os +import time +import pytest +import signal +import subprocess +import coverage.data +import urllib.request +import docker + +from urllib.error import URLError + +from ocs.ocs_client import OCSClient + + +def create_agent_runner_fixture(agent_path, agent_name, args=None): + """Create a pytest fixture for running a given OCS Agent. + + Parameters: + agent_path (str): Relative path to Agent, + i.e. '../agents/fake_data/fake_data_agent.py' + agent_name (str): Short, unique name for the agent + args (list): Additional CLI arguments to add when starting the Agent + + """ + @pytest.fixture() + def run_agent(cov): + env = os.environ.copy() + env['COVERAGE_FILE'] = f'.coverage.agent.{agent_name}' + env['OCS_CONFIG_DIR'] = os.getcwd() + cmd = ['coverage', 'run', + '--rcfile=./.coveragerc', + agent_path, + '--site-file', + './default.yaml'] + if args is not None: + cmd.extend(args) + agentproc = subprocess.Popen(cmd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid) + + # wait for Agent to startup + time.sleep(1) + + yield + + # shutdown Agent + agentproc.send_signal(signal.SIGINT) + time.sleep(1) + + # report coverage + agentcov = coverage.data.CoverageData( + basename=f'.coverage.agent.{agent_name}') + agentcov.read() + # protect against missing --cov flag + if cov is not None: + cov.get_data().update(agentcov) + + return run_agent + + +def create_client_fixture(instance_id, timeout=30): + """Create the fixture that provides tests a Client object. + + Parameters: + instance_id (str): Agent instance-id to connect the Client to + timeout (int): Approximate timeout in seconds for the connection. + Connection attempts will be made X times, with a 1 second pause + between attempts. This is useful if it takes some time for the + Agent to start accepting connections, which varies depending on the + Agent. + + """ + @pytest.fixture() + def client_fixture(): + # Set the OCS_CONFIG_DIR so we read the local default.yaml file + os.environ['OCS_CONFIG_DIR'] = os.getcwd() + print(os.environ['OCS_CONFIG_DIR']) + attempts = 0 + + while attempts < timeout: + try: + client = OCSClient(instance_id) + break + except RuntimeError as e: + print(f"Caught error: {e}") + print("Attempting to reconnect.") + + time.sleep(1) + attempts += 1 + + return client + + return client_fixture + + +def check_crossbar_connection(port=18001, interval=5, max_attempts=6): + """Check that the crossbar server is up and available for an Agent to + connect to. + + Parameters: + port (int): Port the crossbar server is configured to run on for + testing. + interval (float): Amount of time in seconds to wait between checks. + max_attempts (int): Maximum number of attempts before giving up. + + Notes: + For this check to work the crossbar server needs the `Node Info Service + `_ running at the path /info. + + + """ + attempts = 0 + + while attempts < max_attempts: + try: + code = urllib.request.urlopen(f"http://localhost:{port}/info").getcode() + except (URLError, ConnectionResetError): + print("Crossbar server not online yet, waiting 5 seconds.") + time.sleep(interval) + + attempts += 1 + + assert code == 200 + print("Crossbar server online.") diff --git a/tests/integration/test_aggregator_agent_integration.py b/tests/integration/test_aggregator_agent_integration.py index a92078a4..a259f3db 100644 --- a/tests/integration/test_aggregator_agent_integration.py +++ b/tests/integration/test_aggregator_agent_integration.py @@ -3,9 +3,12 @@ import ocs from ocs.base import OpCode -from integration.util import ( +from ocs.testing import ( create_agent_runner_fixture, create_client_fixture, +) + +from integration.util import ( create_crossbar_fixture ) diff --git a/tests/integration/test_fake_data_agent_integration.py b/tests/integration/test_fake_data_agent_integration.py index 5c884e7f..32a2105e 100644 --- a/tests/integration/test_fake_data_agent_integration.py +++ b/tests/integration/test_fake_data_agent_integration.py @@ -3,10 +3,13 @@ import ocs from ocs.base import OpCode -from integration.util import ( +from ocs.testing import ( create_agent_runner_fixture, create_client_fixture, - create_crossbar_fixture +) + +from integration.util import ( + create_crossbar_fixture, ) pytest_plugins = ("docker_compose") diff --git a/tests/integration/test_host_master_agent_integration.py b/tests/integration/test_host_master_agent_integration.py index fb7401a4..265c1a7f 100644 --- a/tests/integration/test_host_master_agent_integration.py +++ b/tests/integration/test_host_master_agent_integration.py @@ -3,9 +3,12 @@ from ocs.base import OpCode -from integration.util import ( +from ocs.testing import ( create_agent_runner_fixture, create_client_fixture, +) + +from integration.util import ( create_crossbar_fixture ) diff --git a/tests/integration/test_influxdb_publisher_integration.py b/tests/integration/test_influxdb_publisher_integration.py index 789f5484..1f634164 100644 --- a/tests/integration/test_influxdb_publisher_integration.py +++ b/tests/integration/test_influxdb_publisher_integration.py @@ -3,9 +3,12 @@ import ocs from ocs.base import OpCode -from integration.util import ( +from ocs.testing import ( create_agent_runner_fixture, create_client_fixture, +) + +from integration.util import ( create_crossbar_fixture ) diff --git a/tests/integration/test_registry_agent_integration.py b/tests/integration/test_registry_agent_integration.py index d4638ed7..028f00b9 100644 --- a/tests/integration/test_registry_agent_integration.py +++ b/tests/integration/test_registry_agent_integration.py @@ -1,9 +1,12 @@ import os import pytest -from integration.util import ( +from ocs.testing import ( create_agent_runner_fixture, create_client_fixture, +) + +from integration.util import ( create_crossbar_fixture ) diff --git a/tests/integration/util.py b/tests/integration/util.py index 8708f215..07b853ce 100644 --- a/tests/integration/util.py +++ b/tests/integration/util.py @@ -1,87 +1,15 @@ -import os -import time import pytest -import signal -import subprocess -import coverage.data -import urllib.request import docker -from urllib.error import URLError - -from ocs.ocs_client import OCSClient - - -def create_agent_runner_fixture(agent_path, agent_name, args=None): - """Create a pytest fixture for running a given OCS Agent. - - Parameters: - agent_path (str): Relative path to Agent, - i.e. '../agents/fake_data/fake_data_agent.py' - agent_name (str): Short, unique name for the agent - args (list): Additional CLI arguments to add when starting the Agent - - """ - @pytest.fixture() - def run_agent(cov): - env = os.environ.copy() - env['COVERAGE_FILE'] = f'.coverage.agent.{agent_name}' - env['OCS_CONFIG_DIR'] = os.getcwd() - cmd = ['coverage', 'run', - '--rcfile=./.coveragerc', - agent_path, - '--site-file', - './default.yaml'] - if args is not None: - cmd.extend(args) - agentproc = subprocess.Popen(cmd, - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=os.setsid) - - # wait for Agent to startup - time.sleep(1) - - yield - - # shutdown Agent - agentproc.send_signal(signal.SIGINT) - time.sleep(1) - - # report coverage - agentcov = coverage.data.CoverageData( - basename=f'.coverage.agent.{agent_name}') - agentcov.read() - # protect against missing --cov flag - if cov is not None: - cov.get_data().update(agentcov) - - return run_agent - - -def _check_crossbar_connection(): - attempts = 0 - - while attempts < 6: - try: - code = urllib.request.urlopen("http://localhost:18001/info").getcode() - except (URLError, ConnectionResetError): - print("Crossbar server not online yet, waiting 5 seconds.") - time.sleep(5) - - attempts += 1 - - assert code == 200 - print("Crossbar server online.") +from ocs.testing import check_crossbar_connection def create_crossbar_fixture(): # Fixture to wait for crossbar server to be available. # Speeds up tests a bit to have this session scoped # If tests start interfering with one another this should be changed to - # "function" scoped and session_scoped_container_getter should be changed to - # function_scoped_container_getter + # "function" scoped and session_scoped_container_getter should be changed + # to function_scoped_container_getter # @pytest.fixture(scope="session") # def wait_for_crossbar(session_scoped_container_getter): @pytest.fixture(scope="function") @@ -90,7 +18,7 @@ def wait_for_crossbar(function_scoped_container_getter): responsive. """ - _check_crossbar_connection() + check_crossbar_connection() return wait_for_crossbar @@ -100,39 +28,4 @@ def restart_crossbar(): client = docker.from_env() crossbar_container = client.containers.get('ocs-tests-crossbar') crossbar_container.restart() - _check_crossbar_connection() - - -def create_client_fixture(instance_id, timeout=30): - """Create the fixture that provides tests a Client object. - - Parameters: - instance_id (str): Agent instance-id to connect the Client to - timeout (int): Approximate timeout in seconds for the connection. - Connection attempts will be made X times, with a 1 second pause - between attempts. This is useful if it takes some time for the - Agent to start accepting connections, which varies depending on the - Agent. - - """ - @pytest.fixture() - def client_fixture(): - # Set the OCS_CONFIG_DIR so we read the local default.yaml file - os.environ['OCS_CONFIG_DIR'] = os.getcwd() - print(os.environ['OCS_CONFIG_DIR']) - attempts = 0 - - while attempts < timeout: - try: - client = OCSClient(instance_id) - break - except RuntimeError as e: - print(f"Caught error: {e}") - print("Attempting to reconnect.") - - time.sleep(1) - attempts += 1 - - return client - - return client_fixture + check_crossbar_connection() From 6db01737823528ff11ff47abcfc5c31f745e79a2 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Sun, 16 Jan 2022 17:10:33 -0500 Subject: [PATCH 2/3] docs: Add testing section to developer's guide --- docs/developer/testing.rst | 37 +++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + tests/README.rst | 12 ++++++------ 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 docs/developer/testing.rst diff --git a/docs/developer/testing.rst b/docs/developer/testing.rst new file mode 100644 index 00000000..97703766 --- /dev/null +++ b/docs/developer/testing.rst @@ -0,0 +1,37 @@ +Testing +======= + +Writing tests for your OCS Agents is important for the long term maintanence of +your Agent. Tests allow other developers to contribute to your Agent and easily +confirm that their changes did not break functionality within the Agent. With +some setup, tests can also allow you to test your Agent without access to the +hardware that it controls. + +Testing within OCS comes in two forms, unit tests and integration tests. Unit +tests test functionality of the Agent code directly, without running the Agent +itself (or any supporting parts, such as the crossbar server, or a piece of +hardware to connect to.) + +Integration tests run a small OCS network, starting up the crossbar server, +your Agent, and any supporting programs that your Agent might need (for +instance, a program accepting serial connections for you Agent to connect to). +As a result, integration tests are more involved than unit tests, requiring +more setup and thus taking longer to execute. + +Both types of testing can be important for fully testing the functionality of +your Agent. + +Running Tests +------------- + +.. include:: ../../tests/README.rst + :start-line: 2 + +Testing API +----------- + +This section details the helper functions within OCS for assisting with testing +your Agents. + +.. automodule:: ocs.testing + :members: diff --git a/docs/index.rst b/docs/index.rst index 9a715854..397cf7bc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ write OCS Agents or Clients. developer/clients developer/data developer/web + developer/testing .. toctree:: diff --git a/tests/README.rst b/tests/README.rst index b3c82c0e..47bd046f 100644 --- a/tests/README.rst +++ b/tests/README.rst @@ -1,5 +1,5 @@ Tests ------ +===== We use pytest as our test runner for OCS. To run all of the tests, from within this ``tests/`` directory, run pytest:: @@ -10,7 +10,7 @@ unit tests will run quickly, however the integration test will take some time, and might require some first time setup. Test Organization -````````````````` +----------------- Tests that test the functionality of the core ocs library are kept in the root of the ``tests/`` directory. They are named identically to the core library module filenames with a ``test_`` prefix. @@ -24,7 +24,7 @@ Finally, integration tests that test the Agents and interaction with them through Clients are kept in ``tests/integration``. Unit Tests -`````````` +---------- The unit tests are built to run quickly and test functionality of individual parts of the OCS library. These are run automatically on every commit to a branch/PR on GitHub, using GitHub Actions. However, you might want to run them @@ -43,7 +43,7 @@ tests, leaving just the unit tests. $ docker run --rm -w="/app/ocs/tests/" ocs sh -c "python3 -m pytest -m 'not integtest'" Integration Tests -````````````````` +----------------- These tests are built to test the running OCS system, and as such need several running components. This includes a crossbar server and each core OCS Agent. In order to run these in an isolated environment we make use of Docker and Docker @@ -70,7 +70,7 @@ all integration tests with:: crossbar on port 18001 instead of port 8001.) Reducing Turnaround Time in Testing -................................... +``````````````````````````````````` Since the integration tests depend on docker containers you need to have the docker images built prior to running the tests. You can build all of the docker images from the root of the ocs repo:: @@ -99,7 +99,7 @@ Agent/container you are working on. For example, in the fake-data-agent:: - "--site-http=http://crossbar:18001/call" Code Coverage -````````````` +------------- Code coverage reports can be produced with the ``--cov`` flag:: python3 -m pytest -m 'not integtest' --cov From 5d683d6061e44e29774bab1e3851120ba740370f Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Tue, 18 Jan 2022 11:57:36 -0500 Subject: [PATCH 3/3] Lint testing.py --- ocs/testing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ocs/testing.py b/ocs/testing.py index 649ed8ac..78c8116a 100644 --- a/ocs/testing.py +++ b/ocs/testing.py @@ -5,7 +5,6 @@ import subprocess import coverage.data import urllib.request -import docker from urllib.error import URLError @@ -40,7 +39,7 @@ def run_agent(cov): stderr=subprocess.PIPE, preexec_fn=os.setsid) - # wait for Agent to startup + # wait for Agent to startup time.sleep(1) yield @@ -107,15 +106,16 @@ def check_crossbar_connection(port=18001, interval=5, max_attempts=6): Notes: For this check to work the crossbar server needs the `Node Info Service - `_ running at the path /info. - + `_ running at the path + /info. """ attempts = 0 while attempts < max_attempts: try: - code = urllib.request.urlopen(f"http://localhost:{port}/info").getcode() + url = f"http://localhost:{port}/info" + code = urllib.request.urlopen(url).getcode() except (URLError, ConnectionResetError): print("Crossbar server not online yet, waiting 5 seconds.") time.sleep(interval)