Merge remote-tracking branch 'upstream/master' into notebook_screencast

* upstream/master:
  MRG, FIX: Speed up I/O tests, mark some slow (mne-tools#7904)
  Proper attribution for Blender tutorial (mne-tools#7900)
  MAINT: Check usage [ci skip] (mne-tools#7902)
  Allow find_bad_channels_maxwell() to return scores (mne-tools#7845)
  Warn if NIRx directory structure has been modified from original format (mne-tools#7898)

.travis.yml

@@ -141,7 +141,7 @@ script:
         pip install -e .;
         python mne/tests/test_evoked.py;
       fi;
-    - echo pytest -m "${CONDITION}" --cov=mne -n 1 ${USE_DIRS}
+    - echo 'pytest -m "${CONDITION}" --cov=mne -n 1 ${USE_DIRS}'
     - pytest -m "${CONDITION}" --tb=short --cov=mne -n 1 ${USE_DIRS}
     # run the minimal one with the testing data
     - if [ "${DEPS}" == "minimal" ]; then

Makefile

@@ -105,10 +105,10 @@ flake:
 	@echo "flake8 passed"
 
 codespell:  # running manually
-	@codespell --builtin clear,rare,informal,names -w -i 3 -q 3 -S $(CODESPELL_SKIPS) --ignore-words=ignore_words.txt $(CODESPELL_DIRS)
+	@codespell --builtin clear,rare,informal,names,usage -w -i 3 -q 3 -S $(CODESPELL_SKIPS) --ignore-words=ignore_words.txt $(CODESPELL_DIRS)
 
 codespell-error:  # running on travis
-	@codespell --builtin clear,rare,informal,names -i 0 -q 7 -S $(CODESPELL_SKIPS) --ignore-words=ignore_words.txt $(CODESPELL_DIRS)
+	@codespell --builtin clear,rare,informal,names,usage -i 0 -q 7 -S $(CODESPELL_SKIPS) --ignore-words=ignore_words.txt $(CODESPELL_DIRS)
 
 pydocstyle:
 	@echo "Running pydocstyle"

azure-pipelines.yml

@@ -252,7 +252,7 @@ jobs:
     displayName: Print NumPy config
   - script: python -c "import mne; mne.datasets.testing.data_path(verbose=True)"
     displayName: 'Get test data'
-  - script: pytest -m "not ultraslowtest" --tb=short --cov=mne -n 1 mne/tests/test_surface.py
+  - script: pytest -m "not slowtest" --tb=short --cov=mne -n 1 mne/tests/test_surface.py
     displayName: 'Run tests'
   - powershell:
       Invoke-WebRequest -Uri https://codecov.io/bash -OutFile codecov.sh

doc/changes/latest.inc

@@ -97,7 +97,7 @@ Changelog
 
 - Add support for reading and writing surfaces in Wavefront .obj format to the :func:`mne.read_surface` and :func:`mne.write_surface` by `Marijn van Vliet`_
 
-- Add tutorial on how to manually fix BEM meshes in Blender by `Marijn van Vliet`_
+- Add tutorial on how to manually fix BEM meshes in Blender by `Marijn van Vliet`_ and `Ezequiel Mikulan`_
 
 Bug
 ~~~

doc/changes/names.inc

@@ -305,3 +305,5 @@
 .. _Lx37: https://github.com/Lx37
 
 .. _Kyle Mathewson: https://github.com/kylemath
+
+.. _Ezequiel Mikulan: https://github.com/ezemikulan

doc/overview/faq.rst

@@ -383,8 +383,8 @@ order of difficulty):
    :ref:`mne watershed_bem`.
 2. Changing the ``--atlas`` and ``--gcaatlas`` options of
    :ref:`mne watershed_bem`.
-3. Manually editing the meshes (see `this tutorial
-   <https://github.com/ezemikulan/blender_freesurfer>`__.
+3. Manually editing the meshes (see :ref:`this tutorial
+   <sphx_glr_auto_tutorials_source-modeling_plot_fix_bem_in_blender.py>`).
 4. Manually running mri_watershed_ with various FreeSurfer flags (e.g.,
    ``-less`` to fix the output).
 5. Going farther back in your Freesurfer pipeline to fix the problem.

ignore_words.txt

@@ -32,3 +32,4 @@ dof
 nwe
 thead
 sherif
+master

mne/beamformer/tests/test_lcmv.py

@@ -428,6 +428,7 @@ def test_make_lcmv(tmpdir, reg, proj):
               noise_cov=noise_cov)
 
 
+@pytest.mark.slowtest
 @pytest.mark.parametrize('weight_norm', (None, 'unit-noise-gain', 'nai'))
 @pytest.mark.parametrize('pick_ori', (None, 'max-power', 'vector'))
 def test_make_lcmv_sphere(pick_ori, weight_norm):

mne/channels/tests/test_interpolation.py

@@ -45,6 +45,7 @@ def _load_data(kind):
 
 @pytest.mark.parametrize('offset', (0., 0.1))
 @pytest.mark.filterwarnings('ignore:.*than 20 mm from head frame origin.*')
+@pytest.mark.slowtest
 def test_interpolation_eeg(offset):
     """Test interpolation of EEG channels."""
     raw, epochs_eeg = _load_data('eeg')
@@ -183,13 +184,14 @@ def _this_interpol(inst, ref_meg=False):
     return inst
 
 
+@pytest.mark.slowtest
 def test_interpolate_meg_ctf():
     """Test interpolation of MEG channels from CTF system."""
     thresh = .85
     tol = .05  # assert the new interpol correlates at least .05 "better"
     bad = 'MLC22-2622'  # select a good channel to test the interpolation
 
-    raw = io.read_raw_fif(raw_fname_ctf, preload=True)  # 3 secs
+    raw = io.read_raw_fif(raw_fname_ctf).crop(0, 1.0).load_data()  # 3 secs
     raw.apply_gradient_compensation(3)
 
     # Show that we have to exclude ref_meg for interpolating CTF MEG-channels

mne/channels/tests/test_montage.py

@@ -336,6 +336,9 @@ def test_montage_readers(
         assert_allclose(actual_ch_pos[kk], expected_ch_pos[kk], atol=1e-5)
     for d1, d2 in zip(dig_montage.dig, expected_dig.dig):
         assert d1['coord_frame'] == d2['coord_frame']
+        for key in ('coord_frame', 'ident', 'kind'):
+            assert isinstance(d1[key], int)
+            assert isinstance(d2[key], int)
 
 
 @testing.requires_testing_data

mne/inverse_sparse/tests/test_mxne_optim.py

@@ -309,6 +309,7 @@ def test_iterative_reweighted_mxne():
     assert_array_equal(X_hat_bcd, X_hat_cd, 5)
 
 
+@pytest.mark.slowtest
 def test_iterative_reweighted_tfmxne():
     """Test convergence of irTF-MxNE solver."""
     M, G, true_active_set = _generate_tf_data()

mne/io/__init__.py

@@ -8,7 +8,7 @@
 from .open import fiff_open, show_fiff, _fiff_get_fid
 from .meas_info import (read_fiducials, write_fiducials, read_info, write_info,
                         _empty_info, _merge_info, _force_update_info, Info,
-                        anonymize_info)
+                        anonymize_info, _writing_info_hdf5)
 
 from .proj import make_eeg_average_ref_proj, Projection
 from .tag import _loc_to_coil_trans, _coil_trans_to_loc, _loc_to_eeg_loc

mne/io/_digitization.py

@@ -67,6 +67,9 @@ def _count_points_by_type(dig):
     )
 
 
+_dig_keys = {'kind', 'ident', 'r', 'coord_frame'}
+
+
 class DigPoint(dict):
     """Container for a digitization point.
 
@@ -90,12 +93,11 @@ class DigPoint(dict):
 
     def __repr__(self):  # noqa: D105
         if self['kind'] == FIFF.FIFFV_POINT_CARDINAL:
-            id_ = _cardinal_kind_rev.get(
-                self.get('ident', -1), 'Unknown cardinal')
+            id_ = _cardinal_kind_rev.get(self['ident'], 'Unknown cardinal')
         else:
             id_ = _dig_kind_proper[
-                _dig_kind_rev.get(self.get('kind', -1), 'unknown')]
-            id_ = ('%s #%s' % (id_, self.get('ident', -1)))
+                _dig_kind_rev.get(self['kind'], 'unknown')]
+            id_ = ('%s #%s' % (id_, self['ident']))
         id_ = id_.rjust(10)
         cf = _coord_frame_name(self['coord_frame'])
         pos = ('(%0.1f, %0.1f, %0.1f) mm' % tuple(1000 * self['r'])).ljust(25)
@@ -115,9 +117,9 @@ def __eq__(self, other):  # noqa: D105
         coordinate frame and position.
         """
         my_keys = ['kind', 'ident', 'coord_frame']
-        if sorted(self.keys()) != sorted(other.keys()):
+        if set(self.keys()) != set(other.keys()):
             return False
-        elif any([self[_] != other[_] for _ in my_keys]):
+        elif any(self[_] != other[_] for _ in my_keys):
             return False
         else:
             return np.allclose(self['r'], other['r'])
@@ -434,7 +436,7 @@ def _make_dig_points(nasion=None, lpa=None, rpa=None, hpi=None,
         except ValueError:  # and if any conversion fails, simply use arange
             idents = np.arange(1, len(dig_ch_pos) + 1)
         for key, ident in zip(dig_ch_pos, idents):
-            dig.append({'r': dig_ch_pos[key], 'ident': ident,
+            dig.append({'r': dig_ch_pos[key], 'ident': int(ident),
                         'kind': FIFF.FIFFV_POINT_EEG,
                         'coord_frame': coord_frame})
 

mne/io/brainvision/brainvision.py

@@ -764,7 +764,7 @@ def _get_vhdr_info(vhdr_fname, eog, misc, scale):
             ch_name=ch_name, coil_type=coil_type, kind=kind, logno=idx + 1,
             scanno=idx + 1, cal=cals[idx], range=ranges[idx],
             loc=np.full(12, np.nan),
-            unit=unit, unit_mul=0.,  # always zero- mne manual pg. 273
+            unit=unit, unit_mul=FIFF.FIFF_UNITM_NONE,
             coord_frame=FIFF.FIFFV_COORD_HEAD))
 
     info._update_redundant()

mne/io/bti/bti.py

@@ -1136,7 +1136,7 @@ def _get_bti_info(pdf_fname, config_fname, head_shape_fname, rotation_x,
         chan_info['ch_name'] = chan_neuromag if rename_channels else chan_4d
         chan_info['logno'] = idx + BTI.FIFF_LOGNO
         chan_info['scanno'] = idx + 1
-        chan_info['cal'] = bti_info['chs'][idx]['scale']
+        chan_info['cal'] = float(bti_info['chs'][idx]['scale'])
 
         if any(chan_4d.startswith(k) for k in ('A', 'M', 'G')):
             loc = bti_info['chs'][idx]['loc']

mne/io/ctf/info.py

@@ -239,7 +239,8 @@ def _convert_channel_info(res4, t, use_eeg_pos):
                 nmeg += 1
                 ch['logno'] = nmeg
             # Encode the software gradiometer order
-            ch['coil_type'] = ch['coil_type'] | (cch['grad_order_no'] << 16)
+            ch['coil_type'] = int(
+                ch['coil_type'] | (cch['grad_order_no'] << 16))
             ch['coord_frame'] = FIFF.FIFFV_COORD_DEVICE
         elif cch['sensor_type_index'] == CTF.CTFV_EEG_CH:
             coord_frame = FIFF.FIFFV_COORD_HEAD

mne/io/edf/edf.py

@@ -412,7 +412,7 @@ def _get_info(fname, stim_channel, eog, misc, exclude, preload):
         chan_info['logno'] = idx + 1
         chan_info['scanno'] = idx + 1
         chan_info['range'] = physical_range
-        chan_info['unit_mul'] = 0.
+        chan_info['unit_mul'] = FIFF.FIFF_UNITM_NONE
         chan_info['ch_name'] = ch_name
         chan_info['unit'] = FIFF.FIFF_UNIT_V
         chan_info['coord_frame'] = FIFF.FIFFV_COORD_HEAD

mne/io/eeglab/tests/test_eeglab.py

@@ -56,6 +56,7 @@ def _check_h5(fname):
 
 @requires_h5py
 @testing.requires_testing_data
+@pytest.mark.slowtest
 @pytest.mark.parametrize(
     'fname', [raw_fname_mat, raw_fname_h5], ids=op.basename
 )

mne/io/egi/egi.py

@@ -245,7 +245,7 @@ def __init__(self, input_fname, eog=None, misc=None,
         sti_ch_idx = [i for i, name in enumerate(ch_names) if
                       name.startswith('STI') or name in event_codes]
         for idx in sti_ch_idx:
-            chs[idx].update({'unit_mul': 0, 'cal': 1,
+            chs[idx].update({'unit_mul': FIFF.FIFF_UNITM_NONE, 'cal': 1.,
                              'kind': FIFF.FIFFV_STIM_CH,
                              'coil_type': FIFF.FIFFV_COIL_NONE,
                              'unit': FIFF.FIFF_UNIT_NONE})

mne/io/egi/egimff.py

@@ -436,7 +436,8 @@ def __init__(self, input_fname, eog=None, misc=None,
         sti_ch_idx = [i for i, name in enumerate(ch_names) if
                       name.startswith('STI') or name in event_codes]
         for idx in sti_ch_idx:
-            chs[idx].update({'unit_mul': 0, 'cal': cals[idx],
+            chs[idx].update({'unit_mul': FIFF.FIFF_UNITM_NONE,
+                             'cal': cals[idx],
                              'kind': FIFF.FIFFV_STIM_CH,
                              'coil_type': FIFF.FIFFV_COIL_NONE,
                              'unit': FIFF.FIFF_UNIT_NONE})

mne/io/fieldtrip/tests/test_fieldtrip.py

@@ -38,6 +38,7 @@
                    'match': NOINFO_WARNING}
 
 
+@pytest.mark.slowtest
 @testing.requires_testing_data
 # Reading the sample CNT data results in a RuntimeWarning because it cannot
 # parse the measurement date. We need to ignore that warning.

mne/io/kit/kit.py

@@ -223,7 +223,7 @@ def _set_stimchannels(self, info, stim, stim_code):
                 cal=KIT.CALIB_FACTOR, logno=nchan, scanno=nchan, range=1.0,
                 unit=FIFF.FIFF_UNIT_NONE, unit_mul=0, ch_name='STI 014',
                 coil_type=FIFF.FIFFV_COIL_NONE, loc=np.full(12, np.nan),
-                kind=FIFF.FIFFV_STIM_CH))
+                kind=FIFF.FIFFV_STIM_CH, coord_frame=FIFF.FIFFV_COORD_UNKNOWN))
             info._update_redundant()
 
         self._raw_extras[0]['stim'] = stim

mne/io/meas_info.py

@@ -7,6 +7,7 @@
 # License: BSD (3-clause)
 
 from collections import Counter
+import contextlib
 from copy import deepcopy
 import datetime
 from io import BytesIO
@@ -538,9 +539,15 @@ def __init__(self, *args, **kwargs):
             _format_trans(self, key)
         for res in self.get('hpi_results', []):
             _format_trans(res, 'coord_trans')
-        if self.get('dig', None) is not None and len(self['dig']) and \
-                not isinstance(self['dig'][0], DigPoint):
-            self['dig'] = _format_dig_points(self['dig'])
+        if self.get('dig', None) is not None and len(self['dig']):
+            if isinstance(self['dig'], dict):  # needs to be unpacked
+                self['dig'] = _dict_unpack(self['dig'], _dig_cast)
+            if not isinstance(self['dig'][0], DigPoint):
+                self['dig'] = _format_dig_points(self['dig'])
+        if isinstance(self.get('chs', None), dict):
+            self['chs']['ch_name'] = [str(x) for x in np.char.decode(
+                self['chs']['ch_name'], encoding='utf8')]
+            self['chs'] = _dict_unpack(self['chs'], _ch_cast)
         for pi, proj in enumerate(self.get('projs', [])):
             if not isinstance(proj, Projection):
                 self['projs'][pi] = Projection(proj)
@@ -2315,3 +2322,40 @@ def _bad_chans_comp(info, ch_names):
         return True, missing_ch_names
 
     return False, missing_ch_names
+
+
+_dig_cast = {'kind': int, 'ident': int, 'r': lambda x: x, 'coord_frame': int}
+_ch_cast = {'scanno': int, 'logno': int, 'kind': int,
+            'range': float, 'cal': float, 'coil_type': int,
+            'loc': lambda x: x, 'unit': int, 'unit_mul': int,
+            'ch_name': lambda x: x, 'coord_frame': int}
+
+
+@contextlib.contextmanager
+def _writing_info_hdf5(info):
+    # Make info writing faster by packing chs and dig into numpy arrays
+    orig_dig = info.get('dig', None)
+    orig_chs = info['chs']
+    try:
+        if orig_dig is not None and len(orig_dig) > 0:
+            info['dig'] = _dict_pack(info['dig'], _dig_cast)
+        info['chs'] = _dict_pack(info['chs'], _ch_cast)
+        info['chs']['ch_name'] = np.char.encode(
+            info['chs']['ch_name'], encoding='utf8')
+        yield
+    finally:
+        if orig_dig is not None:
+            info['dig'] = orig_dig
+        info['chs'] = orig_chs
+
+
+def _dict_pack(obj, casts):
+    # pack a list of dict into dict of array
+    return {key: np.array([o[key] for o in obj]) for key in casts}
+
+
+def _dict_unpack(obj, casts):
+    # unpack a dict of array into a list of dict
+    n = len(obj[list(casts)[0]])
+    return [{key: cast(obj[key][ii]) for key, cast in casts.items()}
+            for ii in range(n)]

mne/io/nirx/nirx.py

@@ -15,6 +15,7 @@
 from ...annotations import Annotations
 from ...transforms import apply_trans, _get_trans
 from ...utils import logger, verbose, fill_doc
+from ...utils import warn
 
 
 @fill_doc
@@ -71,14 +72,19 @@ def __init__(self, fname, preload=False, verbose=None):
 
         # Check if required files exist and store names for later use
         files = dict()
-        keys = ('dat', 'evt', 'hdr', 'inf', 'set', 'tpl', 'wl1', 'wl2',
+        keys = ('evt', 'hdr', 'inf', 'set', 'tpl', 'wl1', 'wl2',
                 'config.txt', 'probeInfo.mat')
         for key in keys:
             files[key] = glob.glob('%s/*%s' % (fname, key))
             if len(files[key]) != 1:
                 raise RuntimeError('Expect one %s file, got %d' %
                                    (key, len(files[key]),))
             files[key] = files[key][0]
+        if len(glob.glob('%s/*%s' % (fname, 'dat'))) != 1:
+            warn("A single dat file was expected in the specified path, but "
+                 "got %d. This may indicate that the file structure has been "
+                 "modified since the measurement was saved." %
+                 (len(glob.glob('%s/*%s' % (fname, 'dat')))))
 
         # Read number of rows/samples of wavelength data
         last_sample = -1

mne/io/nirx/tests/test_nirx.py

@@ -5,6 +5,7 @@
 
 import os.path as op
 import shutil
+import os
 
 import pytest
 from numpy.testing import assert_allclose, assert_array_equal
@@ -36,6 +37,17 @@ def test_nirx_hdr_load():
     assert raw.info['sfreq'] == 12.5
 
 
+@requires_testing_data
+def test_nirx_dat_warn(tmpdir):
+    """Test reading NIRX files when missing data."""
+    shutil.copytree(fname_nirx_15_2_short, str(tmpdir) + "/data/")
+    os.rename(str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.dat",
+              str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.tmp")
+    fname = str(tmpdir) + "/data" + "/NIRS-2019-08-23_001.hdr"
+    with pytest.raises(RuntimeWarning, match='A single dat'):
+        read_raw_nirx(fname, preload=True)
+
+
 @requires_testing_data
 def test_nirx_15_2_short():
     """Test reading NIRX files."""

mne/io/tests/test_raw.py

@@ -17,7 +17,7 @@
 from mne import concatenate_raws, create_info, Annotations
 from mne.datasets import testing
 from mne.externals.h5io import read_hdf5, write_hdf5
-from mne.io import read_raw_fif, RawArray, BaseRaw, Info
+from mne.io import read_raw_fif, RawArray, BaseRaw, Info, _writing_info_hdf5
 from mne.utils import (_TempDir, catch_logging, _raw_annot, _stamp_to_dt,
                        object_diff, check_version)
 from mne.io.meas_info import _get_valid_units
@@ -207,7 +207,8 @@ def _test_raw_reader(reader, test_preloading=True, test_kwargs=True,
     # (all fields should be compatible)
     if check_version('h5py'):
         fname_h5 = op.join(tempdir, 'info.h5')
-        write_hdf5(fname_h5, raw.info)
+        with _writing_info_hdf5(raw.info):
+            write_hdf5(fname_h5, raw.info)
         new_info = Info(read_hdf5(fname_h5))
         assert object_diff(new_info, raw.info) == ''
     return raw