From 9eb12bd9bf78e53970ba11b819c904d2802f43c6 Mon Sep 17 00:00:00 2001 From: Evan Tung Date: Thu, 20 Jun 2024 19:31:36 -0700 Subject: [PATCH] fix: Remove dev_enabled and viz_enabled, use GitHub default python gitignore, remove multiline f-string --- .gitignore | 133 +++++++++++++++++- onair/config/default_config.ini | 2 +- onair/config/reporter_config.ini | 2 - onair/src/run_scripts/execution_engine.py | 16 +-- .../src/run_scripts/test_execution_engine.py | 73 +++++----- 5 files changed, 169 insertions(+), 57 deletions(-) diff --git a/.gitignore b/.gitignore index 3d0969c..f48b391 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,132 @@ -*.pyc -.DS_Store +# Byte-compiled / optimized / DLL files __pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ .coverage -.vscode/ \ No newline at end of file +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# Visual Studio Code +.vscode/ diff --git a/onair/config/default_config.ini b/onair/config/default_config.ini index c107ab0..c5e4a92 100644 --- a/onair/config/default_config.ini +++ b/onair/config/default_config.ini @@ -27,7 +27,7 @@ PlannersPluginDict = {'generic':'plugins/generic/__init__.py'} # Required Key: ComplexPluginDict(s) are used by Agent for complex reasoning ComplexPluginDict = {'generic':'plugins/generic/__init__.py'} -# Required Section: OPTIONS are settable values to change running experience +# Optional Section: OPTIONS are settable values to change running experience [OPTIONS] # Optional Key: IO_Flag denotes whether or not to provide console output # default = false diff --git a/onair/config/reporter_config.ini b/onair/config/reporter_config.ini index 65b3a9b..8eac4a4 100644 --- a/onair/config/reporter_config.ini +++ b/onair/config/reporter_config.ini @@ -19,5 +19,3 @@ ComplexPluginDict = {'Complex Reporter 1':'plugins/reporter', [OPTIONS] IO_Enabled = true -Dev_Enabled = false -Viz_Enabled = false diff --git a/onair/src/run_scripts/execution_engine.py b/onair/src/run_scripts/execution_engine.py index 5f46cd0..6ea946e 100644 --- a/onair/src/run_scripts/execution_engine.py +++ b/onair/src/run_scripts/execution_engine.py @@ -16,7 +16,7 @@ import importlib import ast import shutil -from distutils.dir_util import copy_tree +from shutil import copytree from time import gmtime, strftime from ..run_scripts.sim import Simulator @@ -65,8 +65,7 @@ def parse_configs(self, config_filepath): config = configparser.ConfigParser() if len(config.read(config_filepath)) == 0: - raise FileNotFoundError( - f"Config file at '{config_filepath}' could not be read.") + raise FileNotFoundError(f"Config file at '{config_filepath}' could not be read.") try: # Parse Required Data: FILES @@ -102,8 +101,7 @@ def parse_configs(self, config_filepath): self.IO_Enabled = False except KeyError as e: - new_message = f"Config file: '{ - config_filepath}', missing key: {e.args[0]}" + new_message = f"Config file: '{config_filepath}', missing key: {e.args[0]}" raise KeyError(new_message) from e def parse_plugins_dict(self, config_plugin_dict): @@ -112,13 +110,11 @@ def parse_plugins_dict(self, config_plugin_dict): if isinstance(ast_plugin_dict.body, ast.Dict): temp_plugin_dict = ast.literal_eval(config_plugin_dict) else: - raise ValueError(f"Plugin dict {config_plugin_dict} from { - self.config_filepath} is invalid. It must be a dict.") + raise ValueError(f"Plugin dict {config_plugin_dict} from {self.config_filepath} is invalid. It must be a dict.") for plugin_file in temp_plugin_dict.values(): if not (os.path.exists(plugin_file)): - raise FileNotFoundError(f"In config file '{self.config_filepath}' Plugin path '{ - plugin_file}' does not exist.") + raise FileNotFoundError(f"In config file '{self.config_filepath}' Plugin path '{plugin_file}' does not exist.") return temp_plugin_dict def parse_data(self, parser_file_name, data_file_name, metadata_file_name, subsystems_breakdown=False): @@ -171,7 +167,7 @@ def save_results(self, save_name): save_path = os.environ['ONAIR_SAVE_PATH'] + \ 'saved/' + save_name + '_' + complete_time os.makedirs(save_path, exist_ok=True) - copy_tree(os.environ['ONAIR_TMP_SAVE_PATH'], save_path) + copytree(os.environ['ONAIR_TMP_SAVE_PATH'], save_path) def set_run_param(self, name, val): setattr(self, name, val) diff --git a/test/onair/src/run_scripts/test_execution_engine.py b/test/onair/src/run_scripts/test_execution_engine.py index 2baa4eb..4d4b204 100644 --- a/test/onair/src/run_scripts/test_execution_engine.py +++ b/test/onair/src/run_scripts/test_execution_engine.py @@ -23,7 +23,7 @@ def test_ExecutionEngine__init__sets_expected_values_but_does_no_calls_when_conf arg_run_name = MagicMock() arg_save_flag = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.object(cut, 'init_save_paths') mocker.patch.object(cut, 'parse_configs') @@ -59,7 +59,7 @@ def test_ExecutionEngine__init__does_calls_when_config_file_is_an_occupied_strin arg_run_name = MagicMock() arg_save_flag = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.object(cut, 'init_save_paths') mocker.patch.object(cut, 'parse_configs') @@ -81,7 +81,7 @@ def test_ExecutionEngine__init__does_calls_when_config_file_is_an_occupied_strin def test_ExecutionEngine__init__accepts_no_arguments_using_defaults_instead_with_config_file_default_as_empty_string(mocker): # Arrange - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.object(cut, 'init_save_paths') mocker.patch.object(cut, 'parse_configs') @@ -108,7 +108,7 @@ def test_ExecutionEngine_parse_configs_raises_FileNotFoundError_when_config_cann fake_config_read_result = MagicMock() fake_config_read_result.__len__.return_value = 0 - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.configparser.ConfigParser', return_value=fake_config) @@ -120,8 +120,7 @@ def test_ExecutionEngine_parse_configs_raises_FileNotFoundError_when_config_cann cut.parse_configs(arg_bad_config_filepath) # Assert - assert e_info.match(f"Config file at '{ - arg_bad_config_filepath}' could not be read.") + assert e_info.match(f"Config file at '{arg_bad_config_filepath}' could not be read.") def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_when_the_required_key_FILES_is_not_in_config(mocker): @@ -139,7 +138,7 @@ def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_whe fake_config_read_result = MagicMock() fake_config_read_result.__len__.return_value = 1 - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.configparser.ConfigParser', return_value=fake_config) @@ -190,7 +189,7 @@ def test_ExecutionEngine_parse_configs_raises_KeyError_with_config_file_info_whe fake_config_read_result = MagicMock() fake_config_read_result.__len__.return_value = 1 - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.configparser.ConfigParser', return_value=fake_config) @@ -246,9 +245,6 @@ def test_ExecutionEngine_parse_configs_skips_OPTIONS_when_the_required_section_O fake_learners_plugin_list, fake_planners_plugin_list, fake_complex_plugin_list] - fake_IO_enabled = MagicMock() - fake_Dev_enabled = MagicMock() - fake_Viz_enabled = MagicMock() fake_plugin_dict = MagicMock() fake_keys = MagicMock() fake_plugin = MagicMock() @@ -257,7 +253,7 @@ def test_ExecutionEngine_parse_configs_skips_OPTIONS_when_the_required_section_O fake_keys.__len__.return_value = 1 fake_keys.__iter__.return_value = iter([str(fake_plugin)]) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.configparser.ConfigParser', return_value=fake_config) @@ -333,8 +329,6 @@ def test_ExecutionEngine_parse_configs_sets_all_items_without_error(mocker): fake_planners_plugin_list, fake_complex_plugin_list] fake_IO_enabled = MagicMock() - fake_Dev_enabled = MagicMock() - fake_Viz_enabled = MagicMock() fake_plugin_dict = MagicMock() fake_keys = MagicMock() fake_plugin = MagicMock() @@ -343,7 +337,7 @@ def test_ExecutionEngine_parse_configs_sets_all_items_without_error(mocker): fake_keys.__len__.return_value = 1 fake_keys.__iter__.return_value = iter([str(fake_plugin)]) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.configparser.ConfigParser', return_value=fake_config) @@ -391,7 +385,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_ValueError_when_config_plugin fake_plugin_dict.body = MagicMock() fake_config_filepath = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.config_filepath = fake_config_filepath mocker.patch.object(cut, 'ast_parse_eval', return_value=fake_plugin_dict) @@ -402,8 +396,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_ValueError_when_config_plugin cut.parse_plugins_dict(arg_config_plugin_dict) # Assert - assert e_info.match(f"Plugin dict {arg_config_plugin_dict} from { - fake_config_filepath} is invalid. It must be a dict.") + assert e_info.match(f"Plugin dict {arg_config_plugin_dict} from {fake_config_filepath} is invalid. It must be a dict.") assert cut.ast_parse_eval.call_count == 1 assert cut.ast_parse_eval.call_args_list[0].args == ( arg_config_plugin_dict,) @@ -425,7 +418,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_FileNotFoundError_when_single fake_values.__iter__.return_value = iter([fake_path]) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.config_filepath = fake_config_filepath mocker.patch.object(cut, 'ast_parse_eval', return_value=fake_plugin_dict) @@ -442,8 +435,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_FileNotFoundError_when_single cut.parse_plugins_dict(arg_config_plugin_dict) # Assert - assert e_info.match(f"In config file '{fake_config_filepath}' Plugin path '{ - fake_path}' does not exist.") + assert e_info.match(f"In config file '{fake_config_filepath}' Plugin path '{fake_path}' does not exist.") assert cut.ast_parse_eval.call_count == 1 assert cut.ast_parse_eval.call_args_list[0].args == ( arg_config_plugin_dict,) @@ -478,7 +470,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_FileNotFoundError_when_any_co fake_values.__iter__.return_value = iter([fake_path] * num_fake_items) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.config_filepath = fake_config_filepath mocker.patch.object(cut, 'ast_parse_eval', return_value=fake_plugin_dict) @@ -495,8 +487,7 @@ def test_ExecutionEngine_parse_plugins_list_raises_FileNotFoundError_when_any_co cut.parse_plugins_dict(arg_config_plugin_dict) # Assert - assert e_info.match(f"In config file '{fake_config_filepath}' Plugin path '{ - fake_path}' does not exist.") + assert e_info.match(f"In config file '{fake_config_filepath}' Plugin path '{fake_path}' does not exist.") assert cut.ast_parse_eval.call_count == 1 assert cut.ast_parse_eval.call_args_list[0].args == ( arg_config_plugin_dict,) @@ -524,7 +515,7 @@ def test_ExecutionEngine_returns_empty_dict_when_config_dict_is_empty(mocker): fake_config_filepath = MagicMock() fake_temp_plugin_dict = {} - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.config_filepath = fake_config_filepath mocker.patch.object(cut, 'ast_parse_eval', return_value=fake_plugin_dict) @@ -564,7 +555,7 @@ def test_ExecutionEngine_returns_expected_dict_when_all_mapped_files_exist(mocke fake_values.__iter__.return_value = iter([fake_path] * num_fake_items) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.config_filepath = fake_config_filepath mocker.patch.object(cut, 'ast_parse_eval', return_value=fake_plugin_dict) @@ -612,7 +603,7 @@ def test_ExecutionEngine_parse_data_sets_the_simDataSource_to_a_new_data_source_ fake_module = MagicMock() fake_parser_class_instance = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.importlib.util.spec_from_file_location', return_value=fake_spec) @@ -660,7 +651,7 @@ def __init__(self, data_file, meta_file, subsystems_breakdown): fake_module = MagicMock() fake_parser_class_instance = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.importlib.util.spec_from_file_location', return_value=fake_spec) @@ -683,7 +674,7 @@ def __init__(self, data_file, meta_file, subsystems_breakdown): def test_ExecutionEngine_setup_sim_sets_self_sim_to_new_Simulator(mocker): # Arrange - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.simDataSource = MagicMock() cut.knowledge_rep_plugin_dict = MagicMock() cut.learners_plugin_dict = MagicMock() @@ -712,7 +703,7 @@ def test_ExecutionEngine_setup_sim_sets_self_sim_to_new_Simulator(mocker): def test_ExecutionEngine_run_sim_runs_but_does_not_save_results_when_save_flag_is_False(mocker): # Arrange - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.sim = MagicMock() cut.IO_Enabled = MagicMock() cut.save_flag = False @@ -731,7 +722,7 @@ def test_ExecutionEngine_run_sim_runs_but_does_not_save_results_when_save_flag_i def test_ExecutionEngine_run_sim_runs_and_saves_results_when_save_flag_is_True(mocker): # Arrange - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() cut.sim = MagicMock() cut.IO_Enabled = MagicMock() cut.save_flag = True @@ -760,7 +751,7 @@ def test_ExecutionEngine_init_save_paths_makes_tmp_and_models_and_diagnosis_dire fake_tmp_models_path = str(MagicMock()) fake_tmp_diagnosis_path = str(MagicMock()) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ) mocker.patch(execution_engine.__name__ + '.os.path.join', @@ -799,7 +790,7 @@ def test_ExecutionEngine_delete_save_paths_does_nothing_when_save_path_has_no_tm for i in range(pytest.gen.randint(0, 5)): # 0 to 5 fake_dirs.append(str(MagicMock())) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ) mocker.patch(execution_engine.__name__ + @@ -828,7 +819,7 @@ def test_ExecutionEngine_delete_save_paths_removes_tmp_tree_when_it_exists(mocke for i in range(pytest.gen.randint(0, 5)): # 0 to 5 fake_dirs.append(str(MagicMock())) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ) mocker.patch(execution_engine.__name__ + @@ -862,7 +853,7 @@ def test_ExecutionEngine_delete_save_paths_prints_error_message_when_rmtree_rais for i in range(pytest.gen.randint(0, 5)): # 0 to 5 fake_dirs.append(str(MagicMock())) - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ) mocker.patch(execution_engine.__name__ + @@ -902,7 +893,7 @@ def test_ExecutionEngine_save_results_creates_expected_save_path_and_copies_prop fake_save_path = fake_onair_save_path + 'saved/' + \ arg_save_name + '_' + fake_complete_time - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.gmtime', return_value=fake_gmtime) @@ -910,7 +901,7 @@ def test_ExecutionEngine_save_results_creates_expected_save_path_and_copies_prop return_value=fake_complete_time) mocker.patch.dict(execution_engine.__name__ + '.os.environ', fake_environ) mocker.patch(execution_engine.__name__ + '.os.makedirs') - mocker.patch(execution_engine.__name__ + '.copy_tree') + mocker.patch(execution_engine.__name__ + '.copytree') # Act cut.save_results(arg_save_name) @@ -926,8 +917,8 @@ def test_ExecutionEngine_save_results_creates_expected_save_path_and_copies_prop fake_save_path, ) assert execution_engine.os.makedirs.call_args_list[0].kwargs == { "exist_ok": True} - assert execution_engine.copy_tree.call_count == 1 - assert execution_engine.copy_tree.call_args_list[0].args == ( + assert execution_engine.copytree.call_count == 1 + assert execution_engine.copytree.call_args_list[0].args == ( fake_onair_tmp_save_path, fake_save_path, ) # set_run_param tests @@ -937,7 +928,7 @@ def test_ExecutionEngine_set_run_param_passes_given_arguments_to_setattr(mocker) # Arrange arg_name = MagicMock() arg_val = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + '.setattr') @@ -956,7 +947,7 @@ def test_ExecutionEngine_ast_parse_eval_returns_call_to_ast_parse_with_mode_eval # Arrange arg_config_list = MagicMock() expected_result = MagicMock() - cut = ExecutionEngine.__new__(ExecutionEngine) + cut = ExecutionEngine() mocker.patch(execution_engine.__name__ + ".ast.parse", return_value=expected_result)