Bring up to date with upstream commits

onair/data_handling/parsers/csv_parser.py

@@ -68,7 +68,7 @@ def parse_csv_data(self, dataFile):
  return all_headers, all_data 
 
  def parse_config_data(self, configFile, ss_breakdown):
- parsed_configs = extract_configs(self.metadata_filepath, configFile, csv=True)
+ parsed_configs = extract_configs(self.metadata_filepath, configFile)
  if ss_breakdown == False:
  num_elements = len(parsed_configs['subsystem_assignments'])
  parsed_configs['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]

onair/data_handling/parsers/parser_util.py

@@ -13,7 +13,7 @@
 import datetime
 
 ## Method to extract configuration data and return 3 dictionaries
-def extract_configs(configFilePath, configFile, csv = False):
+def extract_configs(configFilePath, configFile):
  assert configFile != ''
 
  configs = parseTlmConfJson(configFilePath + configFile)
@@ -35,19 +35,6 @@ def extract_configs(configFilePath, configFile, csv = False):
  test_assign[j] = test + limits
 
  return configs
-
-def process_filepath(path, return_config=False, csv = False):
- if csv:
- filename = path.split(os.sep)[-1].replace('_CONFIG', '')
- filename = filename.replace('.txt', '.csv')
- if return_config == True:
- filename = filename.replace('.csv', '_CONFIG.txt')
- return filename
- else:
- filename = path.split(os.sep)[-1].replace('_CONFIG', '')
- if return_config == True:
- filename = filename.replace('.txt', '_CONFIG.txt')
- return filename
 
 def floatify_input(_input, remove_str=False):
  floatified = []

onair/src/data_driven_components/data_driven_learning.py

@@ -11,6 +11,7 @@
 Data driven learning class for managing all data driven AI components
 """
 import importlib.util
+import importlib.util
 
 from ..util.data_conversion import *
 

test/onair/data_handling/parsers/test_csv_parser.py

@@ -372,7 +372,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi
  # Assert
  assert csv_parser.extract_configs.call_count == 1
  assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile)
- assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True}
  assert csv_parser.len.call_count == 0
  assert result == expected_result
 
@@ -400,7 +399,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi
  # Assert
  assert csv_parser.extract_configs.call_count == 1
  assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile)
- assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True}
  assert csv_parser.len.call_count == 1
  assert csv_parser.len.call_args_list[0].args == (fake_empty_processed_filepath, )
  assert result['subsystem_assignments'] == expected_result
@@ -435,7 +433,6 @@ def test_CSV_parse_config_data_returns_call_to_extract_configs_given_metadata_fi
  # Assert
  assert csv_parser.extract_configs.call_count == 1
  assert csv_parser.extract_configs.call_args_list[0].args == (fake_metadata_filepath, arg_configFile)
- assert csv_parser.extract_configs.call_args_list[0].kwargs == {'csv': True}
  assert csv_parser.len.call_count == 1
  assert csv_parser.len.call_args_list[0].args == (fake_processed_filepath, )
  assert result['subsystem_assignments'] == expected_result

test/onair/data_handling/parsers/test_forty_two_parser.py

@@ -252,7 +252,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is
  arg_config_file = MagicMock()
 
  fake_metadata_filepath = MagicMock()
- fake_filename = MagicMock()
  fake_subsystem_assignments = [MagicMock()]
  fake_tests = MagicMock()
  fake_descs = MagicMock()
@@ -264,7 +263,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is
  'description_assignments' : fake_descs}
 
  mocker.patch(forty_two_parser.__name__ + '.extract_configs', return_value=forced_return_extract_configs)
- mocker.patch(forty_two_parser.__name__ + '.process_filepath', return_value=fake_filename)
 
  expected_result = { 'subsystem_assignments' : [['MISSION']],
  'test_assignments' : fake_tests,
@@ -283,7 +281,6 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is
  arg_config_file = MagicMock()
 
  fake_metadata_filepath = MagicMock()
- fake_filename = MagicMock()
  num_ss_assignments = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10
  fake_subsystem_assignments = [MagicMock()] * num_ss_assignments
  fake_tests = MagicMock()
@@ -294,10 +291,8 @@ def test_FortyTwo_parse_config_data_returns_expected_result_when_ss_breakdown_is
  forced_return_extract_configs = { 'subsystem_assignments' : fake_subsystem_assignments,
  'test_assignments' : fake_tests,
  'description_assignments' : fake_descs}
- forced_return_process_filepath = fake_filename
 
  mocker.patch(forty_two_parser.__name__ + '.extract_configs', return_value=forced_return_extract_configs)
- mocker.patch(forty_two_parser.__name__ + '.process_filepath', return_value=forced_return_process_filepath)
 
  expected_result = { 'subsystem_assignments' : [['MISSION']] * num_ss_assignments,
  'test_assignments' : fake_tests,

test/onair/data_handling/parsers/test_parser_util.py

@@ -18,11 +18,10 @@ def test_parser_util_extract_configs_raises_error_when_given_blank_configFile():
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = ''
- arg_csv = MagicMock()
 
  # Act
  with pytest.raises(AssertionError) as e_info:
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert e_info.match('')
@@ -31,7 +30,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock()
- arg_csv = MagicMock()
 
  fake_subsystem_assignments = MagicMock()
  fake_tests = MagicMock()
@@ -47,7 +45,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le
  mocker.patch(parser_util.__name__ + '.str2lst')
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
@@ -61,7 +59,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock()
- arg_csv = MagicMock()
 
  fake_subsystem_assignments = [MagicMock()]
  fake_test_assign = MagicMock()
@@ -82,7 +79,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configs_le
  expected_result['description_assignments'] = fake_descs.copy()
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
@@ -94,7 +91,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock()
- arg_csv = MagicMock()
 
  len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests)
  fake_subsystem_assignments = [MagicMock()] * len_configs
@@ -116,7 +112,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  expected_result['description_assignments'] = fake_descs.copy()
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
@@ -128,7 +124,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock() 
- arg_csv = MagicMock()
 
  len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests)
  num_noops = pytest.gen.randint(2, 10)
@@ -153,7 +148,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  expected_result['description_assignments'] = fake_descs.copy()
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
@@ -165,7 +160,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock()
- arg_csv = MagicMock()
 
  len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests)
  fake_subsystem_assignments = [MagicMock()] * len_configs
@@ -201,7 +195,7 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_len_config
  expected_result['test_assignments'].append(expected_test_assign)
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
@@ -215,7 +209,6 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configFile
  # Arrange
  arg_configFilePath = MagicMock()
  arg_configFile = MagicMock()
- arg_csv = MagicMock()
 
  len_configs = pytest.gen.randint(2, 10) # arbitrary, from 2 to 10 (0 and 1 have own tests)
  fake_subsystem_assignments = []
@@ -254,137 +247,13 @@ def test_parser_util_extract_configs_returns_expected_dicts_dict_when_configFile
  expected_result['description_assignments'] = fake_descs.copy()
 
  # Act
- result = parser_util.extract_configs(arg_configFilePath, arg_configFile, arg_csv)
+ result = parser_util.extract_configs(arg_configFilePath, arg_configFile)
 
  # Assert
  assert parser_util.parseTlmConfJson.call_count == 1
  assert parser_util.parseTlmConfJson.call_args_list[0].args == (arg_configFilePath + arg_configFile, )
  assert result == expected_result
 
-# process_filepath
-def test_parser_util_process_filepath_returns_filename_from_path_with_txt_replaced_by_csv_when_given_csv_resolves_to_True_and_given_return_config_is_not_True(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
- arg_return_config = False if pytest.gen.randint(0, 1) == 1 else 0
- arg_csv = True if pytest.gen.randint(0, 1) == 1 else MagicMock()
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv)
-
- # Assert
- assert result == fake_filename + '.csv'
-
-def test_parser_util_process_filepath_returns_filename_from_path_with_txt_replaced_by__CONFIG_dot_txt_when_given_csv_resolves_to_True_and_given_return_config_is_True(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
- arg_return_config = True
- arg_csv = True if pytest.gen.randint(0, 1) == 1 else MagicMock()
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv)
-
- # Assert
- assert result == fake_filename + '_CONFIG.txt'
-
-def test_parser_util_process_filepath_returns_filename_from_path_when_given_csv_resolves_to_False_and_given_return_config_is_not_True(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
- arg_return_config = False if pytest.gen.randint(0, 1) == 1 else 0
- arg_csv = False if pytest.gen.randint(0, 1) == 1 else 0
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv)
-
- # Assert
- assert result == fake_filename + '.txt'
-
-def test_parser_util_process_filepath_returns_filename_from_path_when_given_csv_resolves_to_False_and_given_return_config_is_True(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
- arg_return_config = True
- arg_csv = False if pytest.gen.randint(0, 1) == 1 else 0
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path, arg_return_config, arg_csv)
-
- # Assert
- assert result == fake_filename + '_CONFIG.txt'
-
-def test_parser_util_process_filepath_default_given_csv_is_False(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
- arg_return_config = True
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path, arg_return_config)
-
- # Assert
- assert result == fake_filename + '_CONFIG.txt'
-
-def test_parser_util_process_filepath_default_given_return_config_is_False(mocker):
- # Arrange
- fake_filename = str(MagicMock())
- fake_os_sep = pytest.gen.choice(chr(47) + chr(92)) # representative separators, 47 = '/', 92 = '\'
-
- arg_path = ""
-
- for i in range(pytest.gen.randint(0, 10)): # arbitrary, from 0 to 10 directories in front of filename
- arg_path += str(MagicMock()) + fake_os_sep
- arg_path += fake_filename + '.txt'
-
- mocker.patch(parser_util.__name__ + '.os.sep', fake_os_sep)
-
- # Act
- result = parser_util.process_filepath(arg_path)
-
- # Assert
- assert result == fake_filename + '.txt'
-
 # floatify_input tests
 def test_parser_util_flotify_input_returns_empty_list_when_given__input_is_vacant(mocker):
  # Arrange