Merge pull request #39 from nasa/37_38_remove_parse_multiple_files_re…

…move_forty_two_parser

37 38 remove parse multiple files remove forty two parser

onair/config/adapter_config.ini

@@ -2,9 +2,9 @@
 
 [DEFAULT]
 TelemetryDataFilePath = data/raw_telemetry_data/
+TelemetryFile = adapter_TLM.txt
 TelemetryMetadataFilePath = data/telemetry_configs/
-MetaFiles = ['adapter_TLM_CONFIG.json']
-TelemetryFiles = ['adapter_TLM.txt']
+MetaFile = adapter_TLM_CONFIG.json
 ParserFileName = forty_two_parser
 ParserName = FortyTwo
 SimName = Adapter

onair/config/default_config.ini

@@ -1,8 +1,8 @@
 [DEFAULT]
 TelemetryDataFilePath = onair/data/raw_telemetry_data/data_physics_generation/Errors
+TelemetryFile = 700_crash_to_earth_1.csv
 TelemetryMetadataFilePath = onair/data/telemetry_configs/
-MetaFiles = ['data_physics_generation_CONFIG.json']
-TelemetryFiles = ['700_crash_to_earth_1.csv']
+MetaFile = data_physics_generation_CONFIG.json
 ParserFileName = csv_parser
 ParserName = CSV
 SimName = CSV

onair/data_handling/parsers/csv_parser.py

@@ -19,63 +19,43 @@
 from .parser_util import * 
 
 class CSV(OnAirParser):
- def pre_process_data(self, dataFiles):
- pass
-
- def process_data_per_data_file(self, data_file):
+ def process_data_file(self, data_file):
  labels, data = self.parse_csv_data(data_file)
- # Header format : { Filename : ['Header', 'Another Header', 'Etc.']}
- self.all_headers[data_file] = labels[data_file] #The key for labels is the file name, so we're able to add that to our "big" dictionary
- # Data format : { 'index': { Filename : ['Data_Point', 'Another Data_Point', 'Etc.']}}
- for key in data:
- # If the key does not exist in the data dictionary already, then data_dict[key][data_file] will give an error
- # In order to skirt this issue, if the key does not exist create an empty dictionary for it, then give it the data file
- if key in self.sim_data:
- self.sim_data[key][data_file] = data[key][data_file]
- else:
- self.sim_data[key] = {}
- self.sim_data[key][data_file] = data[key][data_file]
-
+ self.all_headers = labels
+ self.sim_data = data
 
 ##### INITIAL PROCESSING ####
- def parse_csv_data(self, dataFile):
+ def parse_csv_data(self, data_file):
  #Read in the data set
- dataset = pd.read_csv(os.path.join(self.raw_data_filepath, dataFile), delimiter=',', header=0, dtype=str)
+ dataset = pd.read_csv(data_file, delimiter=',', header=0, dtype=str)
  dataset = dataset.loc[:, ~dataset.columns.str.contains('^Unnamed')]
- #Get headers from dataset and format them into a dictionary
- # {fileName: [Header1, Header2, ...]}
- headersDict = {}
- headersDict[dataFile] = list(dataset.columns.values)
- all_headers = headersDict
+
+ all_headers = list(dataset.columns.values)
  #Find the 'Time' header in the list in order to match 42 file formatting 
  # Converting
- upperCaseStringHeaders = [x.upper().strip() for x in headersDict[dataFile] if isinstance(x, str)]
- #Search for TIME header in list of uppercase string headers, if it's not there it should return a valueerror, set index to -1
- try:
- timeIndex = upperCaseStringHeaders.index('TIME')
- except ValueError:
- timeIndex = -1
+ upperCaseStringHeaders = [x.upper().strip() for x in all_headers if isinstance(x, str)]
+
  #Initialize the entire data dictionary
- all_data = {}
+ all_data = []
  for index, row in dataset.iterrows():
  rowVals = floatify_input(list(row))
- innerStructure = {dataFile : floatify_input(list(row))}
- #If a time header doesn't exist, just assume normal indexing
- if (timeIndex == -1):
- all_data[index] = innerStructure
- else:
- all_data[rowVals[timeIndex]] = innerStructure
+ all_data.append(floatify_input(list(row)))
+
  return all_headers, all_data 
 
- def parse_config_data(self, configFile, ss_breakdown):
- parsed_configs = extract_configs(self.metadata_filepath, configFile)
+ def parse_meta_data_file(self, meta_data_file, ss_breakdown):
+ parsed_configs = extract_meta_data(meta_data_file)
  if ss_breakdown == False:
  num_elements = len(parsed_configs['subsystem_assignments'])
  parsed_configs['subsystem_assignments'] = [['MISSION'] for elem in range(num_elements)]
  return parsed_configs
 
 ##### GETTERS ##################################
-
  def get_sim_data(self):
  return self.all_headers, self.sim_data, self.binning_configs
-
+
+ def get_just_data(self):
+ return self.sim_data
+
+ def get_vehicle_metadata(self):
+ return self.all_headers, self.binning_configs['test_assignments']

onair/data_handling/parsers/on_air_parser.py

@@ -11,52 +11,33 @@
 from .parser_util import * 
 
 class OnAirParser(ABC):
- def __init__(self, rawDataFilepath = '', 
- metadataFilepath = '', 
- dataFiles = '', 
- configFiles = '', 
- ss_breakdown = False):
- """An initial parsing needs to happen in order to use the parser classes
- This means that, if you want to use this class to parse in real time, 
- it needs to at least have seen one sample of the anticipated format """
+ def __init__(self, data_file, meta_file, ss_breakdown = False):
+ """An initial parsing needs to happen in order to use the parser classes
+ This means that, if you want to use this class to parse in real time,
+ it needs to at least have seen one sample of the anticipated format """
 
- self.raw_data_filepath = rawDataFilepath
- self.metadata_filepath = metadataFilepath
- self.all_headers = {}
- self.sim_data = {}
- self.binning_configs = {}
+  self.raw_data_file = data_file
+  self.meta_data_file = meta_file
+  self.all_headers = {}
+  self.sim_data = {}
+  self.binning_configs = {}
 
- if (dataFiles != '') and (configFiles != ''):
- self.pre_process_data(dataFiles)
-
- self.binning_configs['subsystem_assignments'] = {}
- self.binning_configs['test_assignments'] = {}
- self.binning_configs['description_assignments'] = {}
+ configs = self.parse_meta_data_file(self.meta_data_file, ss_breakdown)
+ self.binning_configs['subsystem_assignments'] = configs['subsystem_assignments']
+ self.binning_configs['test_assignments'] = configs['test_assignments']
+ self.binning_configs['description_assignments'] = configs['description_assignments']
 
- configs = self.parse_config_data(str2lst(configFiles)[0], ss_breakdown)
-
- for data_file in str2lst(dataFiles):
- self.process_data_per_data_file(data_file)
- self.binning_configs['subsystem_assignments'][data_file] = configs['subsystem_assignments']
- self.binning_configs['test_assignments'][data_file] = configs['test_assignments']
- self.binning_configs['description_assignments'][data_file] = configs['description_assignments']
-
- @abstractmethod
- def pre_process_data(self, dataFiles):
- """
- Adjust dataFiles before parsing into binning_configs
- """
- raise NotImplementedError
+ self.process_data_file(self.raw_data_file)
 
  @abstractmethod
- def process_data_per_data_file(self, data_file):
+ def process_data_file(self, data_file):
  """
  Adjust each individual data_file before parsing into binning_configs
  """
  raise NotImplementedError
 
  @abstractmethod
- def parse_config_data(self, configFile, ss_breakdown):
+ def parse_meta_data_file(self, meta_data_file, ss_breakdown):
  """
  Create the configs that will be used to populate the binning_configs for the data files
  """

onair/data_handling/parsers/parser_util.py

@@ -13,10 +13,10 @@
 import datetime
 
 ## Method to extract configuration data and return 3 dictionaries
-def extract_configs(configFilePath, configFile):
- assert configFile != ''
+def extract_meta_data(meta_data_file):
+ assert meta_data_file != ''
 
- configs = parseTlmConfJson(configFilePath + configFile)
+ configs = parseTlmConfJson(meta_data_file)
 
  configs_len = len(configs['subsystem_assignments'])