FIX: don't use op.realpath when reading raw (#9227)

* don't use op.realpath when reading raw

* BUG: Fix which

* FIX: Test

Co-authored-by: Eric Larson <larson.eric.d@gmail.com>

mne/annotations.py

@@ -782,7 +782,7 @@ def read_annotations(fname, sfreq='auto', uint16_codec=None):
     _validate_type(fname, 'path-like', 'fname')
     fname = _check_fname(
         fname, overwrite='read', must_exist=True,
-        allow_dir=str(fname).endswith('.ds'),  # allow_dir for CTF
+        need_dir=str(fname).endswith('.ds'),  # for CTF
         name='fname')
     name = op.basename(fname)
     if name.endswith(('fif', 'fif.gz')):

mne/io/artemis123/artemis123.py

@@ -8,7 +8,7 @@
 import calendar
 
 from .utils import _load_mne_locs, _read_pos
-from ...utils import logger, warn, verbose
+from ...utils import logger, warn, verbose, _check_fname
 from ..utils import _read_segments_file
 from ..base import BaseRaw
 from ..meas_info import _empty_info
@@ -308,6 +308,7 @@ def __init__(self, input_fname, preload=False, verbose=None,
         from scipy.spatial.distance import cdist
         from ...chpi import (compute_chpi_amplitudes, compute_chpi_locs,
                              _fit_coil_order_dev_head_trans)
+        input_fname = _check_fname(input_fname, 'read', True, 'input_fname')
         fname, ext = op.splitext(input_fname)
         if ext == '.txt':
             input_fname = fname + '.bin'

mne/io/base.py

@@ -1394,7 +1394,7 @@ def save(self, fname, picks=None, tmin=0, tmax=None, buffer_size_sec=None,
         or all forms of SSS). It is recommended not to concatenate and
         then save raw files for this reason.
         """
-        fname = op.realpath(fname)
+        fname = op.abspath(fname)
         endings = ('raw.fif', 'raw_sss.fif', 'raw_tsss.fif',
                    '_meg.fif', '_eeg.fif', '_ieeg.fif')
         endings += tuple([f'{e}.gz' for e in endings])

mne/io/boxy/boxy.py

@@ -9,7 +9,7 @@
 from ..base import BaseRaw
 from ..meas_info import create_info
 from ..utils import _mult_cal_one
-from ...utils import logger, verbose, fill_doc
+from ...utils import logger, verbose, fill_doc, _check_fname
 from ...annotations import Annotations
 
 
@@ -65,6 +65,7 @@ def __init__(self, fname, preload=False, verbose=None):
         raw_extras = dict()
         raw_extras['offsets'] = list()  # keep track of our offsets
         sfreq = None
+        fname = _check_fname(fname, 'read', True, 'fname')
         with open(fname, 'r') as fid:
             line_num = 0
             i_line = fid.readline()

mne/io/ctf/ctf.py

@@ -6,13 +6,12 @@
 # License: BSD (3-clause)
 
 import os
-import os.path as op
 
 import numpy as np
 
 from .._digitization import _format_dig_points
 from ...utils import (verbose, logger, _clean_names, fill_doc, _check_option,
-                      _validate_type)
+                      _check_fname)
 
 from ..base import BaseRaw
 from ..utils import _mult_cal_one, _blk_read_lims
@@ -91,13 +90,11 @@ class RawCTF(BaseRaw):
     def __init__(self, directory, system_clock='truncate', preload=False,
                  verbose=None, clean_names=False):  # noqa: D102
         # adapted from mne_ctf2fiff.c
-        _validate_type(directory, 'path-like', 'directory')
-        directory = str(directory)
+        directory = _check_fname(directory, 'read', True, 'directory',
+                                 need_dir=True)
         if not directory.endswith('.ds'):
             raise TypeError('directory must be a directory ending with ".ds", '
                             f'got {directory}')
-        if not op.isdir(directory):
-            raise ValueError('directory does not exist: "%s"' % directory)
         _check_option('system_clock', system_clock, ['ignore', 'truncate'])
         logger.info('ds directory : %s' % directory)
         res4 = _read_res4(directory)  # Read the magical res4 file

mne/io/ctf/tests/test_ctf.py

@@ -220,12 +220,14 @@ def test_read_ctf(tmpdir):
             assert_allclose(raw.annotations.onset, [2.15])
             assert_allclose(raw.annotations.duration, [0.0225])
 
-    pytest.raises(TypeError, read_raw_ctf, 1)
-    pytest.raises(ValueError, read_raw_ctf, ctf_fname_continuous + 'foo.ds')
+    with pytest.raises(TypeError, match='path-like'):
+        read_raw_ctf(1)
+    with pytest.raises(FileNotFoundError, match='does not exist'):
+        read_raw_ctf(ctf_fname_continuous + 'foo.ds')
     # test ignoring of system clock
     read_raw_ctf(op.join(ctf_dir, ctf_fname_continuous), 'ignore')
-    pytest.raises(ValueError, read_raw_ctf,
-                  op.join(ctf_dir, ctf_fname_continuous), 'foo')
+    with pytest.raises(ValueError, match='system_clock'):
+        read_raw_ctf(op.join(ctf_dir, ctf_fname_continuous), 'foo')
 
 
 @testing.requires_testing_data

mne/io/curry/curry.py

@@ -63,7 +63,7 @@ def _get_curry_file_structure(fname, required=()):
     """Store paths to a dict and check for required files."""
     _msg = "The following required files cannot be found: {0}.\nPlease make " \
            "sure all required files are located in the same directory as {1}."
-    _check_fname(fname, overwrite='read', must_exist=True)
+    fname = _check_fname(fname, 'read', True, 'fname')
 
     # we don't use os.path.splitext to also handle extensions like .cdt.dpa
     fname_base, ext = fname.split(".", maxsplit=1)

mne/io/eeglab/eeglab.py

@@ -12,7 +12,7 @@
 from ..constants import FIFF
 from ..meas_info import create_info
 from ..base import BaseRaw
-from ...utils import logger, verbose, warn, fill_doc, Bunch
+from ...utils import logger, verbose, warn, fill_doc, Bunch, _check_fname
 from ...channels import make_dig_montage
 from ...epochs import BaseEpochs
 from ...event import read_events
@@ -22,7 +22,7 @@
 CAL = 1e-6
 
 
-def _check_fname(fname, dataname):
+def _check_eeglab_fname(fname, dataname):
     """Check whether the filename is valid.
 
     Check if the file extension is ``.fdt`` (older ``.dat`` being invalid) or
@@ -314,6 +314,7 @@ class RawEEGLAB(BaseRaw):
     @verbose
     def __init__(self, input_fname, eog=(),
                  preload=False, uint16_codec=None, verbose=None):  # noqa: D102
+        input_fname = _check_fname(input_fname, 'read', True, 'input_fname')
         eeg = _check_load_mat(input_fname, uint16_codec)
         if eeg.trials != 1:
             raise TypeError('The number of trials is %d. It must be 1 for raw'
@@ -325,7 +326,7 @@ def __init__(self, input_fname, eog=(),
 
         # read the data
         if isinstance(eeg.data, str):
-            data_fname = _check_fname(input_fname, eeg.data)
+            data_fname = _check_eeglab_fname(input_fname, eeg.data)
             logger.info('Reading %s' % data_fname)
 
             super(RawEEGLAB, self).__init__(
@@ -510,7 +511,7 @@ def __init__(self, input_fname, events=None, event_id=None, tmin=0,
                                  '(event id %i)' % (key, val))
 
         if isinstance(eeg.data, str):
-            data_fname = _check_fname(input_fname, eeg.data)
+            data_fname = _check_eeglab_fname(input_fname, eeg.data)
             with open(data_fname, 'rb') as data_fid:
                 data = np.fromfile(data_fid, dtype=np.float32)
                 data = data.reshape((eeg.nbchan, eeg.pnts, eeg.trials),

mne/io/egi/egi.py

@@ -14,7 +14,7 @@
 from ..utils import _read_segments_file, _create_chs
 from ..meas_info import _empty_info
 from ..constants import FIFF
-from ...utils import verbose, logger, warn, _validate_type
+from ...utils import verbose, logger, warn, _validate_type, _check_fname
 
 
 def _read_header(fid):
@@ -152,7 +152,6 @@ def read_raw_egi(input_fname, eog=None, misc=None,
     """
     _validate_type(input_fname, 'path-like', 'input_fname')
     input_fname = str(input_fname)
-
     if input_fname.endswith('.mff'):
         return _read_raw_egi_mff(input_fname, eog, misc, include,
                                  exclude, preload, channel_naming, verbose)
@@ -167,6 +166,7 @@ class RawEGI(BaseRaw):
     def __init__(self, input_fname, eog=None, misc=None,
                  include=None, exclude=None, preload=False,
                  channel_naming='E%d', verbose=None):  # noqa: D102
+        input_fname = _check_fname(input_fname, 'read', True, 'input_fname')
         if eog is None:
             eog = []
         if misc is None:

mne/io/egi/egimff.py

@@ -19,7 +19,7 @@
 from ..proj import setup_proj
 from ..utils import _create_chs, _mult_cal_one
 from ...annotations import Annotations
-from ...utils import verbose, logger, warn, _check_option
+from ...utils import verbose, logger, warn, _check_option, _check_fname
 from ...evoked import EvokedArray
 
 
@@ -398,6 +398,8 @@ def __init__(self, input_fname, eog=None, misc=None,
                  include=None, exclude=None, preload=False,
                  channel_naming='E%d', verbose=None):
         """Init the RawMff class."""
+        input_fname = _check_fname(input_fname, 'read', True, 'input_fname',
+                                   need_dir=True)
         logger.info('Reading EGI MFF Header from %s...' % input_fname)
         egi_info = _read_header(input_fname)
         if eog is None:

mne/io/eximia/eximia.py

@@ -8,7 +8,7 @@
 from ..base import BaseRaw
 from ..utils import _read_segments_file, _file_size
 from ..meas_info import create_info
-from ...utils import logger, verbose, warn, fill_doc
+from ...utils import logger, verbose, warn, fill_doc, _check_fname
 
 
 @fill_doc
@@ -52,6 +52,7 @@ class RawEximia(BaseRaw):
 
     @verbose
     def __init__(self, fname, preload=False, verbose=None):
+        fname = _check_fname(fname, 'read', True, 'fname')
         data_name = op.basename(fname)
         logger.info('Loading %s' % data_name)
         # Create vhdr and vmrk files so that we can use mne_brain_vision2fiff

mne/io/fiff/raw.py

@@ -25,7 +25,7 @@
 
 from ...event import AcqParserFIF
 from ...utils import (check_fname, logger, verbose, warn, fill_doc, _file_like,
-                      _on_missing)
+                      _on_missing, _check_fname)
 
 
 @fill_doc
@@ -75,13 +75,13 @@ class Raw(BaseRaw):
     def __init__(self, fname, allow_maxshield=False, preload=False,
                  on_split_missing='raise', verbose=None):  # noqa: D102
         raws = []
-        do_check_fname = not _file_like(fname)
+        do_check_ext = not _file_like(fname)
         next_fname = fname
         while next_fname is not None:
             raw, next_fname, buffer_size_sec = \
                 self._read_raw_file(next_fname, allow_maxshield,
-                                    preload, do_check_fname)
-            do_check_fname = False
+                                    preload, do_check_ext)
+            do_check_ext = False
             raws.append(raw)
             if next_fname is not None:
                 if not op.exists(next_fname):
@@ -132,19 +132,19 @@ def __init__(self, fname, allow_maxshield=False, preload=False,
 
     @verbose
     def _read_raw_file(self, fname, allow_maxshield, preload,
-                       do_check_fname=True, verbose=None):
+                       do_check_ext=True, verbose=None):
         """Read in header information from a raw file."""
         logger.info('Opening raw data file %s...' % fname)
 
         #   Read in the whole file if preload is on and .fif.gz (saves time)
         if not _file_like(fname):
-            if do_check_fname:
+            if do_check_ext:
                 endings = ('raw.fif', 'raw_sss.fif', 'raw_tsss.fif',
                            '_meg.fif', '_eeg.fif', '_ieeg.fif')
                 endings += tuple([f'{e}.gz' for e in endings])
                 check_fname(fname, 'raw', endings)
             # filename
-            fname = op.realpath(fname)
+            fname = _check_fname(fname, 'read', True, 'fname')
             ext = os.path.splitext(fname)[1].lower()
             whole_file = preload if '.gz' in ext else False
             del ext

mne/io/fiff/tests/test_raw_fiff.py

@@ -11,6 +11,7 @@
 import os.path as op
 import pathlib
 import pickle
+import shutil
 import sys
 
 import numpy as np
@@ -28,7 +29,7 @@
                  compute_proj_raw, pick_types, pick_channels, create_info,
                  pick_info)
 from mne.utils import (requires_pandas, assert_object_equal, _dt_to_stamp,
-                       requires_mne, run_subprocess, run_tests_if_main,
+                       requires_mne, run_subprocess,
                        assert_and_remove_boundary_annot)
 from mne.annotations import Annotations
 
@@ -1361,12 +1362,16 @@ def test_add_channels():
 @testing.requires_testing_data
 def test_save(tmpdir):
     """Test saving raw."""
-    raw = read_raw_fif(fif_fname, preload=False)
+    temp_fname = tmpdir.join('test_raw.fif')
+    shutil.copyfile(fif_fname, temp_fname)
+    raw = read_raw_fif(temp_fname, preload=False)
     # can't write over file being read
-    pytest.raises(ValueError, raw.save, fif_fname)
-    raw = read_raw_fif(fif_fname, preload=True)
+    with pytest.raises(ValueError, match='to the same file'):
+        raw.save(temp_fname)
+    raw.load_data()
     # can't overwrite file without overwrite=True
-    pytest.raises(IOError, raw.save, fif_fname)
+    with pytest.raises(IOError, match='file exists'):
+        raw.save(fif_fname)
 
     # test abspath support and annotations
     orig_time = _dt_to_stamp(raw.info['meas_date'])[0] + raw._first_time
@@ -1720,4 +1725,22 @@ def test_bad_acq(fname):
             assert tag == ent
 
 
-run_tests_if_main()
+@pytest.mark.skipif(sys.platform not in ('darwin', 'linux'),
+                    reason='Needs proper symlinking')
+def test_split_symlink(tmpdir):
+    """Test split files with symlinks."""
+    # regression test for gh-9221
+    first = str(tmpdir.mkdir('first').join('test_raw.fif'))
+    raw = read_raw_fif(fif_fname).pick('meg').load_data()
+    raw.save(first, buffer_size_sec=1, split_size='10MB', verbose=True)
+    second = first[:-4] + '-1.fif'
+    assert op.isfile(second)
+    assert not op.isfile(first[:-4] + '-2.fif')
+    new_first = tmpdir.mkdir('a').join('test_raw.fif')
+    new_second = tmpdir.mkdir('b').join('test_raw-1.fif')
+    shutil.move(first, new_first)
+    shutil.move(second, new_second)
+    os.symlink(new_first, first)
+    os.symlink(new_second, second)
+    raw_new = read_raw_fif(first)
+    assert_allclose(raw_new.get_data(), raw.get_data())

mne/io/nedf/nedf.py

@@ -10,7 +10,7 @@
 from ..base import BaseRaw
 from ..meas_info import create_info
 from ..utils import _mult_cal_one
-from ...utils import warn, verbose
+from ...utils import warn, verbose, _check_fname
 
 
 def _getsubnodetext(node, name):
@@ -131,6 +131,7 @@ class RawNedf(BaseRaw):
     """Raw object from NeuroElectrics nedf file."""
 
     def __init__(self, filename, preload=False, verbose=None):
+        filename = _check_fname(filename, 'read', True, 'filename')
         with open(filename, mode='rb') as fid:
             header = fid.read(_HDRLEN)
         header, dt, dt_last, n_samp, n_full = _parse_nedf_header(header)

mne/io/nihon/nihon.py

@@ -8,7 +8,7 @@
 
 import numpy as np
 
-from ...utils import fill_doc, logger, verbose, warn
+from ...utils import fill_doc, logger, verbose, warn, _check_fname
 from ..base import BaseRaw
 from ..meas_info import create_info
 from ...annotations import Annotations
@@ -308,6 +308,7 @@ class RawNihon(BaseRaw):
 
     @verbose
     def __init__(self, fname, preload=False, verbose=None):
+        fname = _check_fname(fname, 'read', True, 'fname')
         fname = _ensure_path(fname)
         data_name = fname.name
         logger.info('Loading %s' % data_name)

mne/io/nirx/nirx.py

@@ -16,7 +16,8 @@
 from ..meas_info import create_info, _format_dig_points
 from ...annotations import Annotations
 from ...transforms import apply_trans, _get_trans
-from ...utils import logger, verbose, fill_doc, warn
+from ...utils import (logger, verbose, fill_doc, warn, _check_fname,
+                      _validate_type)
 
 
 @fill_doc
@@ -69,12 +70,12 @@ def __init__(self, fname, preload=False, verbose=None):
         from ...externals.pymatreader import read_mat
         from ...coreg import get_mni_fiducials  # avoid circular import prob
         logger.info('Loading %s' % fname)
-
+        _validate_type(fname, 'path-like', 'fname')
+        fname = str(fname)
         if fname.endswith('.hdr'):
             fname = op.dirname(op.abspath(fname))
 
-        if not op.isdir(fname):
-            raise FileNotFoundError('The path you specified does not exist.')
+        fname = _check_fname(fname, 'read', True, 'fname', need_dir=True)
 
         # Check if required files exist and store names for later use
         files = dict()

mne/io/nirx/tests/test_nirx.py

@@ -46,7 +46,7 @@ def test_nirx_hdr_load():
 @requires_testing_data
 def test_nirx_missing_warn():
     """Test reading NIRX files when missing data."""
-    with pytest.raises(FileNotFoundError, match='The path you'):
+    with pytest.raises(FileNotFoundError, match='does not exist'):
         read_raw_nirx(fname_nirx_15_2_short + "1", preload=True)