IO: misc updates from welib

Author: ebranlard

pyFAST/input_output/converters.py

@@ -4,7 +4,44 @@
 # --------------------------------------------------------------------------------
 # --- Writing pandas DataFrame to different formats
 # --------------------------------------------------------------------------------
-# The
+def writeDataFrameToFormat(df, filename, fformat):
+    """  
+    Write a dataframe to disk based on user-specified fileformat
+    - df: pandas dataframe
+    - filename: filename 
+    - fformat: fileformat in: ['csv', 'outb', 'parquet']
+    """
+
+    if fformat=='outb':
+        dataFrameToOUTB(df, filename)
+    elif fformat=='parquet':
+        dataFrameToParquet(df, filename)
+    elif fformat=='csv':
+        dataFrameToCSV(df, filename, sep=',', index=False)
+    else:
+        raise Exception('File format not supported for dataframe export `{}`'.format(fformat))
+
+def writeDataFrameAutoFormat(df, filename, fformat=None):
+    """ 
+    Write a dataframe to disk based on extension
+    - df: pandas dataframe
+    - filename: filename 
+    """
+    if fformat is not None:
+        raise Exception()
+    base, ext = os.path.splitext(filename)
+    ext = ext.lower()
+    if ext in ['.outb']:
+        fformat = 'outb'
+    elif ext in ['.parquet']:
+        fformat = 'parquet'
+    elif ext in ['.csv']:
+        fformat = 'csv'
+    else:
+        print('[WARN] defaulting to csv, extension unknown: `{}`'.format(ext))
+        fformat = 'csv'
+
+    writeDataFrameToFormat(df, filename, fformat)
 
 def writeFileDataFrames(fileObject, writer, extension='.conv', filename=None, **kwargs):
     """ 
@@ -35,7 +72,6 @@ def writeFileDataFrames(fileObject, writer, extension='.conv', filename=None, **
     else:
         writeDataFrame(df=dfs, writer=writer, filename=filename, **kwargs)
 
-
 def writeDataFrame(df, writer, filename, **kwargs):
     """ 
     Write a dataframe to disk based on a "writer" function. 
@@ -47,16 +83,10 @@ def writeDataFrame(df, writer, filename, **kwargs):
 
 # --- Low level writers
 def dataFrameToCSV(df, filename, sep=',', index=False, **kwargs):
-    base, ext = os.path.splitext(filename)
-    if len(ext)==0:
-        filename = base='.csv'
     df.to_csv(filename, sep=sep, index=index, **kwargs)
 
 def dataFrameToOUTB(df, filename, **kwargs):
     from .fast_output_file import writeDataFrame as writeDataFrameToOUTB
-    base, ext = os.path.splitext(filename)
-    if len(ext)==0:
-        filename = base='.outb'
     writeDataFrameToOUTB(df, filename, binary=True)
 
 def dataFrameToParquet(df, filename, **kwargs):

pyFAST/input_output/excel_file.py

@@ -75,8 +75,10 @@ def __repr__(self):
 
 
     def _toDataFrame(self):
-        #cols=['Alpha_[deg]','Cl_[-]','Cd_[-]','Cm_[-]']
-        #dfs[name] = pd.DataFrame(data=..., columns=cols)
-        #df=pd.DataFrame(data=,columns=) 
-        return self.data
+        if len(self.data)==1:
+            # Return a single dataframe
+            return self.data[list(self.data.keys())[0]]
+        else:
+            # Return dictionary
+            return self.data
 

pyFAST/input_output/fast_output_file.py

@@ -235,23 +235,30 @@ def isBinary(filename):
 
 
 
-def load_ascii_output(filename, method='numpy'):
+def load_ascii_output(filename, method='numpy', encoding='ascii'):
 
 
     if method in ['forLoop','pandas']:
         from .file import numberOfLines
         nLines = numberOfLines(filename, method=2)
 
-    with open(filename) as f:
+    with open(filename, encoding=encoding, errors='ignore') as f:
         info = {}
         info['name'] = os.path.splitext(os.path.basename(filename))[0]
         # Header is whatever is before the keyword `time`
-        in_header = True
         header = []
-        while in_header:
+        maxHeaderLines=35
+        headerRead = False
+        for i in range(maxHeaderLines):
             l = f.readline()
             if not l:
                 raise Exception('Error finding the end of FAST out file header. Keyword Time missing.')
+            # Check for utf-16
+            if l[:3] == '\x00 \x00':
+                f.close()
+                encoding=''
+                print('[WARN] Attempt to re-read the file with encoding utf-16')
+                return load_ascii_output(filename=filename, method=method, encoding='utf-16')
             first_word = (l+' dummy').lower().split()[0]
             in_header=  (first_word != 'time') and  (first_word != 'alpha')
             if in_header:
@@ -260,6 +267,10 @@ def load_ascii_output(filename, method='numpy'):
                 info['description'] = header
                 info['attribute_names'] = l.split()
                 info['attribute_units'] = [unit[1:-1] for unit in f.readline().split()]
+                headerRead=True
+                break
+        if not headerRead:
+            raise WrongFormatError('Could not find the keyword "Time" or "Alpha" in the first {} lines of the file'.format(maxHeaderLines))
 
         nHeader = len(header)+1
         nCols = len(info['attribute_names'])
@@ -285,13 +296,13 @@ def load_ascii_output(filename, method='numpy'):
             data = np.zeros((nRows, nCols))
             for i in range(nRows):
                 l = f.readline().strip()
-                sp = np.array(l.split()).astype(np.float)
+                sp = np.array(l.split()).astype(float)
                 data[i,:] = sp[:nCols]
 
         elif method == 'listCompr':
             # --- Method 4 - List comprehension
             # Data, up to end of file or empty line (potential comment line at the end)
-            data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(np.float)
+            data = np.array([l.strip().split() for l in takewhile(lambda x: len(x.strip())>0, f.readlines())]).astype(float)
         else:
             raise NotImplementedError()
 

pyFAST/input_output/file_formats.py

@@ -11,6 +11,8 @@ def isRightFormat(fileformat, filename, **kwargs):
     except WrongFormatError:
         return False,None
     except:
+        # We Raise the Exception.  
+        # It's the responsability of all the file classes to Return a WrongFormatError
         raise
 
 class FileFormat():

pyFAST/input_output/flex_out_file.py

@@ -134,7 +134,7 @@ def read_flex_res(filename, dtype=np.float32):
 
 
 def read_flex_sensor(sensor_file):
-    with open(sensor_file, encoding="utf-8") as fid:
+    with open(sensor_file, 'r') as fid:
         sensor_info_lines = fid.readlines()[2:]
     sensor_info = []
     d=dict({ 'ID':[],'Gain':[],'Offset':[],'Unit':[],'Name':[],'Description':[]});

pyFAST/input_output/matlabmat_file.py

@@ -63,7 +63,6 @@ def read(self, filename=None, **kwargs):
             raise EmptyFileError('File is empty:',self.filename)
 
         mfile = scipy.io.loadmat(self.filename)
-        import pdb; pdb.set_trace()
 
     def write(self, filename=None):
         """ Rewrite object to file, or write object to `filename` if provided """

pyFAST/input_output/pickle_file.py

@@ -65,6 +65,12 @@ def _setData(self, data):
             else:
                 self['data'] = data
 
+    def addDict(self, data):
+        self._setData(data)
+
+    def additem(self, key, data):
+        self[key]=data
+
     def read(self, filename=None, **kwargs):
         """ Reads the file self.filename, or `filename` if provided """
         # --- Standard tests and exceptions (generic code)

pyFAST/input_output/wetb/hawc2/htc_file.py

@@ -591,7 +591,6 @@ def unix_path(self, filename):
 if "__main__" == __name__:
     f = HTCFile(r"C:/Work/BAR-Local/Hawc2ToBeamDyn/sim.htc", ".")
     print(f.input_files())
-    import pdb; pdb.set_trace()
 #     f.save(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT_power_curve.htc")
 #
 #     f = HTCFile(r"C:\mmpe\HAWC2\models\DTU10MWRef6.0\htc\DTU_10MW_RWT.htc", "../")