Merge branch 'main' into 464-prime

CHANGES.rst

@@ -8,6 +8,13 @@ Change History
 
 :1.4.0:  release expected 2021-01-31
 
+    * `#472 <https://github.com/BCDA-APS/apstools/pull/472>`_
+       Respond to changes in upstream packages.
+
+       * package requirements
+       * auto-detection of command list format (Excel or text)
+       * use *openpyxl* instead of *xlrd* and *pandas* to read Excel
+
     * `#470 <https://github.com/BCDA-APS/apstools/pull/470>`_
        Area Detector plugin preparation & detection.
 

apstools/plans.py

@@ -201,12 +201,13 @@ def get_command_list(filename):
         raise IOError(f"file not found: {filename}")
     try:
         commands = parse_Excel_command_file(filename)
-    except APS_utils.ExcelReadError:
+    except (ValueError, APS_utils.ExcelReadError):
         try:
             commands = parse_text_command_file(filename)
         except ValueError as exc:
-            emsg = f"could not read {filename} as command list file: {exc}"
-            raise CommandFileReadError(emsg)
+            raise CommandFileReadError(
+                f"could not read {filename} as command list file: {exc}"
+            )
     return commands
 
 

apstools/utils.py

@@ -65,9 +65,10 @@
 import json
 import logging
 import math
+import openpyxl
+import openpyxl.utils.exceptions
 import ophyd
 import os
-import pandas
 import psutil
 import pyRestTable
 import re
@@ -77,7 +78,6 @@
 import threading
 import time
 import warnings
-import xlrd
 import zipfile
 
 from .filewriters import _rebuild_scan_command
@@ -88,7 +88,7 @@
 MAX_EPICS_STRINGOUT_LENGTH = 40
 
 
-class ExcelReadError(xlrd.XLRDError): ...
+class ExcelReadError(openpyxl.utils.exceptions.InvalidFileException): ...
 
 
 def cleanupText(text):
@@ -1059,30 +1059,36 @@ def handleExcelRowEntry(self, entry):       # subclass MUST override
     def parse(self, labels_row_num=None, data_start_row_num=None, ignore_extra=True):
         labels_row_num = labels_row_num or self.LABELS_ROW
         try:
+            wb = openpyxl.load_workbook(self.fname)
+            ws = wb.worksheets[self.sheet_name]
             if ignore_extra:
                 # ignore data outside of table in spreadsheet file
-                nrows, ncols = self.getTableBoundaries(labels_row_num)
-                xl = pandas.read_excel(
-                    self.fname,
-                    sheet_name=self.sheet_name,
-                    skiprows=labels_row_num,
-                    usecols=range(ncols),
-                    nrows=nrows,
-                    )
+                data = list(ws.rows)[labels_row_num:]
+                self.data_labels = []
+                for c in data[0]:
+                    if c.value is None:
+                        break
+                    self.data_labels.append(c.value)
+                rows = []
+                for r in data[1:]:
+                    if r[0].value is None:
+                        break
+                    rows.append(r[:len(self.data_labels)])
             else:
-                xl = pandas.read_excel(
-                    self.fname,
-                    sheet_name=self.sheet_name,
-                    header=None,
-                    )
-        except xlrd.XLRDError as exc:
+                # use the whole sheet
+                rows = list(ws.rows)
+                # create the column titles
+                self.data_labels = [
+                    f"Column_{i+1}"
+                    for i in range(len(rows[0]))
+                ]
+        except openpyxl.utils.exceptions.InvalidFileException as exc:
             raise ExcelReadError(exc)
-        self.data_labels = list(map(str, xl.columns.values))
         # unused: data_start_row_num = data_start_row_num or labels_row_num+1
-        for row_data in xl.values:
+        for row in rows:
             entry = OrderedDict()
             for _col, label in enumerate(self.data_labels):
-                entry[label] = self._getExcelColumnValue(row_data, _col)
+                entry[label] = row[_col].value
                 self.handle_single_entry(entry)
             self.handleExcelRowEntry(entry)
 
@@ -1101,29 +1107,6 @@ def _isExcel_nan(self, value):
             return False
         return math.isnan(value)
 
-    def getTableBoundaries(self, labels_row_num=None):
-        """
-        identify how many rows and columns are in the Excel spreadsheet table
-        """
-        labels_row_num = labels_row_num or self.LABELS_ROW
-        xl = pandas.read_excel(self.fname, sheet_name=self.sheet_name, skiprows=labels_row_num)
-
-        ncols = len(xl.columns)
-        for i, k in enumerate(xl.columns):
-            if k.startswith(f"Unnamed: {i}"):
-                # TODO: verify all values under this label are NaN
-                ncols = i
-                break
-
-        nrows = len(xl.values)
-        for j, r in enumerate(xl.values):
-            r = r[:ncols]
-            if False not in [self._isExcel_nan(value) for value in r]:
-                nrows = j
-                break
-
-        return nrows, ncols
-
 
 class ExcelDatabaseFileGeneric(ExcelDatabaseFileBase):
     """

conda-recipe/meta.yaml

@@ -36,17 +36,16 @@ requirements:
     - python
     - pip
   run:
-    - python>=3.6.0
+    - python >=3.6.0
     - aps-dm-api
-    - bluesky>=1.6.2
-    - databroker>=1.0.6
+    - bluesky >=1.6.2
+    - databroker >=1.0.6
     - h5py
-    - ophyd>=1.5.1
-    - pandas
-    - pyEpics>=3.4.2
+    - ophyd >=1.5.1
+    - openpyxl
+    - pyEpics >=3.4.2
     - pyRestTable
-    - spec2nexus>=2021.1.7
-    - xlrd
+    - spec2nexus >=2021.1.7
 
 test:
   imports:

docs/source/conf.py

@@ -175,7 +175,7 @@
     'h5py',
     'matplotlib',
     'networkx',
-    'pandas',
+    'openpyxl',
     'pyRestTable',
     'snapshot',
     'xarray',

environment.yml

@@ -5,22 +5,23 @@ channels:
   - aps-anl-tag
   - nsls2forge
 dependencies:
-  - python>=3.6.0
+  - python >=3.6.0
   - aps-dm-api
-  - bluesky>=1.6.2
-  - databroker>=1.0.6
+  - bluesky >=1.6.2
+  - databroker >=1.0.6
   - h5py
   - hklpy
-  - ophyd>=1.5.1
+  - openpyxl
+  - ophyd >=1.5.1
   - pip
-  - pyEpics>=3.4.2
-  - pyqt=5
+  - pyEpics >=3.4.2
+  - pyqt =5
   - pyRestTable
-  - qt=5
-  - spec2nexus>=2021.1.7
+  - qt =5
+  - spec2nexus >=2021.1.7
   - sphinx
-  - xlrd
   - pip:
+    - bluesky_live
     - ipython-genutils==0.2.0
     - prompt-toolkit==1.0.15
     - sphinx-rtd-theme

requirements.txt

@@ -3,7 +3,7 @@ databroker>=1.0.6
 h5py
 numpy
 ophyd>=1.5.1
-pandas
+openpyxl
 pyepics>=3.4.2
 pyRestTable
 spec2nexus>=2021.1.7

tests/test_commandlist.py

@@ -33,15 +33,15 @@ def test_ExcelFile(self):
         ====== ====== =========================================
         line # action parameters
         ====== ====== =========================================
-        1      row1   91, 26.0, 85.0, None, blank, 8.0
-        2      row2   9, 39.0, 29.0, 85.0, sample, 60.0
-        3      row3   54, None, 38.0, 3.0, blank, 76.0
-        4      row4   71, 36.0, 95.0, 83.0, foil, 12.0
-        5      row5   55, 75.0, 59.0, 84.0, DNA, 34.0
-        6      row6   18, 49.0, 31.0, 34.0, lecithin, 47.0
+        1      row1   91, 26, 85, None, blank, 8
+        2      row2   9, 39, 29, 85, sample, 60
+        3      row3   54, None, 38, 3, blank, 76
+        4      row4   71, 36, 95, 83, foil, 12
+        5      row5   55, 75, 59, 84, DNA, 34
+        6      row6   18, 49, 31, 34, lecithin, 47
         7      row7   37, None, None, None, a big mix  of stuff
-        8      row8   37, 80.0, 79.0, 45.0, salt water, 36.0
-        9      row9   72, 98.0, 67.0, 89.0, surprises, 49.0
+        8      row8   37, 80, 79, 45, salt water, 36
+        9      row9   72, 98, 67, 89, surprises, 49
         ====== ====== =========================================
         """.strip()
         self.compare_tables_as_str(expected, received)
@@ -59,17 +59,17 @@ def test_ExcelCommandList(self):
         table = APS_utils.command_list_as_table(commands, show_raw=False)
         received = str(table).strip()
         expected = """
-        ====== ============ =============================
+        ====== ============ ===========================
         line # action       parameters
-        ====== ============ =============================
+        ====== ============ ===========================
         1      mono_shutter open
-        2      USAXSscan    45.07, 98.3, 0.0, Water Blank
-        3      saxsExp      45.07, 98.3, 0.0, Water Blank
-        4      waxwsExp     45.07, 98.3, 0.0, Water Blank
-        5      USAXSscan    12, 12.0, 1.2, plastic
-        6      USAXSscan    12, 37.0, 0.1, Al foil
+        2      USAXSscan    45.07, 98.3, 0, Water Blank
+        3      saxsExp      45.07, 98.3, 0, Water Blank
+        4      waxwsExp     45.07, 98.3, 0, Water Blank
+        5      USAXSscan    12, 12, 1.2, plastic
+        6      USAXSscan    12, 37, 0.1, Al foil
         7      mono_shutter close
-        ====== ============ =============================
+        ====== ============ ===========================
         """.strip()
         self.compare_tables_as_str(expected, received)