From e9611da459e9fdfcd63f6646a7aa03c817fbf5f5 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Mon, 16 Aug 2021 11:38:58 -0400 Subject: [PATCH 1/5] Protect spt3g/so3g imports when testing aggregator --- requirements.txt | 1 + tests/test_aggregator.py | 24 +++++++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1da22d8f..1d9d7ced 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,3 +25,4 @@ pytest pytest-cov docker pytest-docker-compose +pytest-dependency diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py index 8cd7eb56..82df3e7c 100644 --- a/tests/test_aggregator.py +++ b/tests/test_aggregator.py @@ -4,11 +4,25 @@ from unittest.mock import patch -import so3g -from spt3g import core +try: + import so3g + from spt3g import core -from ocs.agent.aggregator import Provider, g3_cast, make_filename + from ocs.agent.aggregator import Provider, g3_cast, make_filename +except ModuleNotFoundError as e: + print(f"Unable to import either so3g or spt3g: {e}") + +@pytest.mark.dependency(name="so3g") +def test_so3g_spt3g_import(): + """Test that we can import spt3g/so3g. Used to skip tests dependent on + these imports. + + """ + import so3g + from spt3g import core + +@pytest.mark.dependency(depends=["so3g"]) def test_passing_float_in_provider_to_frame(): """Float is the expected type we should be passing. @@ -31,6 +45,7 @@ def test_passing_float_in_provider_to_frame(): provider.to_frame(hksess=sess) +@pytest.mark.dependency(depends=["so3g"]) def test_passing_float_like_str_in_provider_to_frame(): """Here we test passing a string amongst ints. This shouldn't make it to the aggregator, and instead the Aggregator logs should display an error @@ -55,6 +70,7 @@ def test_passing_float_like_str_in_provider_to_frame(): provider.to_frame(hksess=sess) +@pytest.mark.dependency(depends=["so3g"]) def test_passing_non_float_like_str_in_provider_to_frame(): """Similar to passing a float like str, here we test passing a non-float like str. We can't put this into an so3g.IrregBlockDouble(), so this'll fail. @@ -80,6 +96,7 @@ def test_passing_non_float_like_str_in_provider_to_frame(): provider.to_frame(hksess=sess) +@pytest.mark.dependency(depends=["so3g"]) def test_sparsely_sampled_block(): """If a block is sparsely sampled and published, the aggregator was including its block_name anyway, even when missing. This test publishes two @@ -346,6 +363,7 @@ def test_enforced_field_which_becomes_empty(): assert 'invalid_field_123' in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_g3_cast(): correct_tests = [ ([1, 2, 3, 4], core.G3VectorInt), From 8dcc5c4bdc16887ccee0f4b58e65c161eb7b7c39 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Mon, 16 Aug 2021 11:46:38 -0400 Subject: [PATCH 2/5] Add numpy to requirements file --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 1d9d7ced..bbbdf065 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,9 @@ msgpack influxdb PyYAML +# FakeDataAgent +numpy + # Lakeshore240 pyserial From 3e3a4684d3e396bbcc87018f2a097888bdd39fd3 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Mon, 16 Aug 2021 11:47:34 -0400 Subject: [PATCH 3/5] Protect so3g import when testing crossbar integration --- tests/test_crossbar_integration.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/test_crossbar_integration.py b/tests/test_crossbar_integration.py index 4e2f24a2..ec505456 100644 --- a/tests/test_crossbar_integration.py +++ b/tests/test_crossbar_integration.py @@ -9,13 +9,25 @@ import numpy as np from ocs.matched_client import MatchedClient -from so3g import hk + +try: + from so3g import hk +except ModuleNotFoundError as e: + print(f"Unable to import so3g: {e}") # Set OCS_CONFIG_DIR environment variable os.environ['OCS_CONFIG_DIR'] = os.getcwd() pytest_plugins = ("docker_compose",) +@pytest.mark.dependency(name="so3g") +def test_so3g_spt3g_import(): + """Test that we can import so3g. Used to skip tests dependent on + this import. + + """ + import so3g + # Fixture to wait for crossbar server to be available. @pytest.fixture(scope="function") def wait_for_crossbar(function_scoped_container_getter): @@ -90,6 +102,7 @@ def test_influxdb_publisher_after_crossbar_restart(wait_for_crossbar): """ pass +@pytest.mark.dependency(depends=["so3g"]) @pytest.mark.integtest def test_aggregator_after_crossbar_restart(wait_for_crossbar): """Test that the aggregator reconnects after a crossbar restart and that From e4c534dc8e47118e7b65e8a61dcd52a544db826e Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Mon, 16 Aug 2021 11:54:18 -0400 Subject: [PATCH 4/5] Mark tests on aggregator classes/functions as dependent on so3g --- tests/test_aggregator.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py index 82df3e7c..853d6c26 100644 --- a/tests/test_aggregator.py +++ b/tests/test_aggregator.py @@ -169,6 +169,7 @@ def test_sparsely_sampled_block(): # This is perhaps another problem, I'm passing irregular length data sets and # it's not raising any sort of alarm. How does this get handled? +@pytest.mark.dependency(depends=["so3g"]) def test_data_type_in_provider_save_to_block(): provider = Provider('test_provider', 'test_sessid', 3, 1) provider.frame_start_time = time.time() @@ -181,6 +182,7 @@ def test_data_type_in_provider_save_to_block(): provider.save_to_block(data) # 'data' field names +@pytest.mark.dependency(depends=["so3g"]) def test_passing_invalid_data_field_name1(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -203,6 +205,7 @@ def test_passing_invalid_data_field_name1(): assert 'invalidkey' in provider.blocks['test'].data.keys() assert 'invalid.key' not in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_passing_invalid_data_field_name2(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -225,6 +228,7 @@ def test_passing_invalid_data_field_name2(): assert '__invalidkey' in provider.blocks['test'].data.keys() assert '__123invalid.key' not in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_passing_too_long_data_field_name(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -248,6 +252,7 @@ def test_passing_too_long_data_field_name(): assert 'a'*255 in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_long_duplicate_name(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -272,6 +277,7 @@ def test_long_duplicate_name(): assert 'a'*255 in provider.blocks['test'].data.keys() assert 'a'*252 + '_01' in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_reducing_to_duplicate_field_names(): """Invalid data field names get modified by the Aggregator to comply with the set rules. This can result in duplicate field names under certain @@ -299,6 +305,7 @@ def test_reducing_to_duplicate_field_names(): assert 'aninvalidkey' in provider.blocks['test'].data.keys() assert 'aninvalidkey_01' in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_space_replacement_in_field_names(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -320,6 +327,7 @@ def test_space_replacement_in_field_names(): assert '_an_invalid_key' in provider.blocks['test'].data.keys() assert 'key2' in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_empty_field_name(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -340,6 +348,7 @@ def test_empty_field_name(): assert '' not in provider.blocks['test'].data.keys() +@pytest.mark.dependency(depends=["so3g"]) def test_enforced_field_which_becomes_empty(): """Invalid data field names should get caught by the Feed, however, we check for them in the Aggregator as well. @@ -386,6 +395,7 @@ def test_g3_cast(): g3_cast(x) +@pytest.mark.dependency(depends=["so3g"]) def test_make_filename_directory_creation(tmpdir): """make_filename() should be able to create directories to store the .g3 files in. @@ -397,6 +407,7 @@ def test_make_filename_directory_creation(tmpdir): os.path.isdir(os.path.basename(fname)) +@pytest.mark.dependency(depends=["so3g"]) def test_make_filename_directory_creation_no_subdirs(tmpdir): """make_filename() should raise a FileNotFoundError if make_subdirs is False. @@ -407,6 +418,7 @@ def test_make_filename_directory_creation_no_subdirs(tmpdir): make_filename(test_dir, make_subdirs=False) +@pytest.mark.dependency(depends=["so3g"]) @patch('os.makedirs', side_effect=PermissionError('mocked permission error')) def test_make_filename_directory_creation_permissions(tmpdir): """make_filename() should raise a PermissionError if it runs into one when From 88ae0c099848783d01ef63e1dc7f2d208d072765 Mon Sep 17 00:00:00 2001 From: Brian Koopman Date: Mon, 16 Aug 2021 11:59:47 -0400 Subject: [PATCH 5/5] Lint new tests --- tests/test_aggregator.py | 4 ++++ tests/test_crossbar_integration.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tests/test_aggregator.py b/tests/test_aggregator.py index 853d6c26..a9a0322a 100644 --- a/tests/test_aggregator.py +++ b/tests/test_aggregator.py @@ -22,6 +22,10 @@ def test_so3g_spt3g_import(): import so3g from spt3g import core + # Just to avoid flake8 complaining we aren't using these imports + print(so3g.__file__) + print(core.__file__) + @pytest.mark.dependency(depends=["so3g"]) def test_passing_float_in_provider_to_frame(): """Float is the expected type we should be passing. diff --git a/tests/test_crossbar_integration.py b/tests/test_crossbar_integration.py index ec505456..927ca380 100644 --- a/tests/test_crossbar_integration.py +++ b/tests/test_crossbar_integration.py @@ -28,6 +28,9 @@ def test_so3g_spt3g_import(): """ import so3g + # Just to prevent flake8 from complaining + print(so3g.__file__) + # Fixture to wait for crossbar server to be available. @pytest.fixture(scope="function") def wait_for_crossbar(function_scoped_container_getter):