Fix for string columns in FITS files

CHANGES.md

@@ -14,6 +14,10 @@ v0.8 (unreleased)
 v0.7.2 (unreleased)
 -------------------
 
+* Fix a bug that caused string columns in FITS files to not be read
+  correctly, and updated coerce_numeric to give a ValueError for string
+  columns that can't be convered.
+
 * Make sure main window title is set. [#914]
 
 v0.7.1 (2016-03-29)

glue/core/component.py

@@ -196,6 +196,7 @@ def autotyped(cls, data, units=None):
             return CategoricalComponent(data, units=units)
 
         n = coerce_numeric(data)
+
         thresh = 0.5
         try:
             use_categorical = np.issubdtype(data.dtype, np.character) and \

glue/core/data_factories/fits.py

@@ -119,7 +119,7 @@ def new_data():
                     if column.ndim != 1:
                         warnings.warn("Dropping column '{0}' since it is not 1-dimensional".format(column_name))
                         continue
-                    component = Component(column, units=column.unit)
+                    component = Component.autotyped(column, units=column.unit)
                     data.add_component(component=component,
                                        label=column_name)
     return [groups[idx] for idx in groups]

glue/core/data_factories/npy.py

@@ -37,7 +37,7 @@ def npy_reader(filename, format='auto', auto_merge=False, **kwargs):
 
     d = Data()
     for name in npy_data.dtype.names:
-        comp = Component(npy_data[name])
+        comp = Component.autotyped(npy_data[name])
         d.add_component(comp, label=name)
 
     return d
@@ -79,7 +79,7 @@ def npz_reader(filename, format='auto', auto_merge=False, **kwargs):
                              " arrays, e.g., with specified names.")
 
         for name in arr.dtype.names:
-            comp = Component(arr[name])
+            comp = Component.autotyped(arr[name])
             d.add_component(comp, label=name)
 
         groups.append(d)

glue/core/data_factories/tests/test_numpy.py

@@ -16,8 +16,9 @@ def test_npy_load(tmpdir):
         f.seek(0)
 
         data2 = df.load_data(f.name)
-        for name in data.dtype.names:
-            assert_array_equal(data[name], data2[name])
+        assert_array_equal(data['name'], data2.get_component('name').labels)
+        assert_array_equal(data['ra'], data2['ra'])
+        assert_array_equal(data['dec'], data2['dec'])
 
 def test_npz_load(tmpdir):
     data1 = np.array([("a",152.2352,-21.513), ("b",21.412,35.1341)],
@@ -26,15 +27,18 @@ def test_npz_load(tmpdir):
                      dtype=[('name','|S1'),('l','f8'),('b','f8')])
 
     with open(tmpdir.join('test.npz').strpath, 'wb') as f:
+
         np.savez(f, data1=data1, data2=data2)
         f.seek(0)
 
         data_loaded = df.load_data(f.name)
 
         arr = data_loaded[0]
-        for name in data1.dtype.names:
-            assert_array_equal(data1[name], arr[name])
+        assert_array_equal(data1['name'], arr.get_component('name').labels)
+        assert_array_equal(data1['ra'], arr['ra'])
+        assert_array_equal(data1['dec'], arr['dec'])
 
         arr = data_loaded[1]
-        for name in data2.dtype.names:
-            assert_array_equal(data2[name], arr[name])
+        assert_array_equal(data2['name'], arr.get_component('name').labels)
+        assert_array_equal(data2['l'], arr['l'])
+        assert_array_equal(data2['b'], arr['b'])

glue/utils/array.py

@@ -97,7 +97,10 @@ def coerce_numeric(arr):
         return arr.astype(np.int)
 
     # a string dtype, or anything else
-    return pd.Series(arr).convert_objects(convert_numeric=True).values
+    try:
+        return pd.to_numeric(arr, errors='coerce')
+    except AttributeError:  # older versions of pandas
+        return pd.Series(arr).convert_objects(convert_numeric=True).values
 
 
 def check_sorted(array):