From 417fa3e9e8b2febe51ba834a1502ba76bd977244 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 10 Jul 2018 15:19:37 -0400 Subject: [PATCH 01/14] ENH: Test constant consistency --- mne/io/constants.py | 4 +- mne/io/tests/test_constants.py | 170 +++++++++++++++++++++++++++++++++ 2 files changed, 172 insertions(+), 2 deletions(-) create mode 100644 mne/io/tests/test_constants.py diff --git a/mne/io/constants.py b/mne/io/constants.py index 9f2ac012a1a..eedbf4742a2 100644 --- a/mne/io/constants.py +++ b/mne/io/constants.py @@ -26,7 +26,7 @@ def __setattr__(self, attr, val): # noqa: D105 # FIFF version number in use # FIFF.FIFFC_MAJOR_VERSION = 1 -FIFF.FIFFC_MINOR_VERSION = 3 +FIFF.FIFFC_MINOR_VERSION = 4 FIFF.FIFFC_VERSION = FIFF.FIFFC_MAJOR_VERSION << 16 | FIFF.FIFFC_MINOR_VERSION # @@ -129,7 +129,7 @@ def __setattr__(self, attr, val): # noqa: D105 FIFF.FIFF_ARTEF_REMOVAL = 221 FIFF.FIFF_COORD_TRANS = 222 FIFF.FIFF_HIGHPASS = 223 -FIFF.FIFF_CH_CALS = 22 # This will not occur in new files +FIFF.FIFF_CH_CALS = 224 # This will not occur in new files FIFF.FIFF_HPI_BAD_CHS = 225 # List of channels considered to be bad in hpi FIFF.FIFF_HPI_CORR_COEFF = 226 # HPI curve fit correlations FIFF.FIFF_EVENT_COMMENT = 227 # Comment about the events used in averaging diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py new file mode 100644 index 00000000000..6546c6b352d --- /dev/null +++ b/mne/io/tests/test_constants.py @@ -0,0 +1,170 @@ +# Author: Eric Larson +# +# License: BSD (3-clause) + +import os.path as op +import shutil +import zipfile + +from mne.io.constants import FIFF +from mne.utils import _fetch_file, requires_good_network + + +iod_dups = (355, 359) # these are in both MEGIN and MNE files +tag_bads = (299,) # no proper dtype or units (sss_expansion) +# XXX -> ask MEGIN folks what it should be +tag_dups = (3501, 3507) # in both MEGIN and MNE files + + +_dir_ignore_names = ('clear', 'copy', 'fromkeys', 'get', 'items', 'keys', + 'pop', 'popitem', 'setdefault', 'update', 'values') + + +@requires_good_network +def test_constants(tmpdir): + """Test compensation.""" + dest = op.join(tmpdir, 'fiff.zip') + _fetch_file('https://api.github.com/repos/mne-tools/fiff-constants' + '/zipball/master', dest) + names = list() + with zipfile.ZipFile(dest, 'r') as ff: + for name in ff.namelist(): + if 'Dictionary' in name: + ff.extract(name, tmpdir) + names.append(op.basename(name)) + shutil.move(op.join(tmpdir, name), op.join(tmpdir, names[-1])) + names = sorted(names) + assert names == ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt', + 'DictionaryStructures.txt', + 'DictionaryTags.txt', 'DictionaryTags_MNE.txt', + 'DictionaryTypes.txt', 'DictionaryTypes_MNE.txt'] + # IOD (MEGIN and MNE) + iod = dict() + fiff_version = None + for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']: + with open(op.join(tmpdir, name), 'rb') as fid: + for line in fid: + line = line.decode('ISO-8859-1').strip() + if line.startswith('# Packing revision'): + assert fiff_version is None + fiff_version = line.split()[-1] + if (line.startswith('#') or line.startswith('alias') or + len(line) == 0): + continue + line = line.split('"') + assert len(line) in (1, 2, 3) + desc = '' if len(line) == 1 else line[1] + line = line[0].split() + assert len(line) in (2, 3) + if len(line) == 2: + kind, id_ = line + else: + kind, id_, tagged = line + assert tagged in ('tagged',) + id_ = int(id_) + if id_ not in iod_dups: + assert id_ not in iod + iod[id_] = [kind, desc] + # Tags (MEGIN) + tags = dict() + with open(op.join(tmpdir, 'DictionaryTags.txt'), 'rb') as fid: + for line in fid: + line = line.decode('ISO-8859-1').strip() + if (line.startswith('#') or line.startswith('alias') or + line.startswith(':') or len(line) == 0): + continue + line = line.split('"') + assert len(line) in (1, 2, 3), line + desc = '' if len(line) == 1 else line[1] + line = line[0].split() + if int(line[1]) in tag_bads: + assert len(line) == 2 + kind, id_ = line + dtype, unit = '-', '-' + else: + assert len(line) == 4, line + kind, id_, dtype, unit = line + id_ = int(id_) + val = [kind, dtype, unit] + assert id_ not in tags, (tags.get(id_), val) + tags[id_] = val + # Tags (MNE) + with open(op.join(tmpdir, 'DictionaryTags_MNE.txt'), 'rb') as fid: + for li, line in enumerate(fid): + line = line.decode('ISO-8859-1').strip() + # ignore continuation lines (*) + if (line.startswith('#') or line.startswith('alias') or + line.startswith(':') or line.startswith('*') or + len(line) == 0): + continue + # weird syntax around line 80: + if line in ('/*', '"'): + continue + line = line.split('"') + assert len(line) in (1, 2, 3), line + if len(line) == 3 and len(line[2]) > 0: + l2 = line[2].strip() + assert l2.startswith('/*') and l2.endswith('*/'), l2 + desc = '' if len(line) == 1 else line[1] + line = line[0].split() + assert len(line) == 3, (li + 1, line) + kind, id_, dtype = line + unit = '-' + id_ = int(id_) + val = [kind, dtype, unit] + if id_ not in tag_dups: + assert id_ not in tags, (tags.get(id_), val) + tags[id_] = val + + # + # Assertions + # + + # Version + mne_version = '%d.%d' % (FIFF.FIFFC_MAJOR_VERSION, + FIFF.FIFFC_MINOR_VERSION) + assert fiff_version == mne_version + unknowns = list() + + # Assert that all our constants are in the dict + # (we are not necessarily complete the other way) + for name in sorted(dir(FIFF)): + if name.startswith('_') or name in _dir_ignore_names: + continue + val = getattr(FIFF, name) + if name.startswith('FIFFC_'): + # Checked above + assert name in ('FIFFC_MAJOR_VERSION', 'FIFFC_MINOR_VERSION', + 'FIFFC_VERSION') + elif name.startswith('FIFFB_'): + # We put these in the tag section rather than the IOD/block section + # -> XXX fix this by moving them to the IOD files in fiff-constants + if name in ('FIFFB_MNE_ANNOTATIONS', + 'FIFFB_MNE_EPOCHS_DROP_LOG', + 'FIFFB_MNE_EPOCHS_SELECTION', + 'FIFFB_MNE_METADATA'): + assert val in tags + continue + assert val in iod, (val, name) + elif name.startswith('FIFFT_'): + continue # XXX add check for this + elif name.startswith('FIFFV_'): + continue # XXX add check for this + elif name.startswith('FIFF_UNIT_'): + continue # XXX add check for this + elif name.startswith('FIFF_UNITM_'): # multipliers + continue # XXX add check for this + elif name.startswith('FWD_'): + # These are not FIFF constants really + # -> XXX remove from FIFF to forward.py namespace? + continue + elif name.startswith('FIFF_'): + if name == 'FIFF_MNE_KIT_SYSTEM_ID': + # This is missing + # -> XXX add this to Tags_MNE + continue + assert val in tags, (name, val) + else: + unknowns.append((name, val)) + unknowns = '\n\t'.join('%s (%s)' % u for u in unknowns) + assert len(unknowns) == 0, 'Unknown types\n\t%s' % unknowns From 06d80f7c7081f59584264ab0a04cf0154b6fb112 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 10 Jul 2018 16:06:43 -0400 Subject: [PATCH 02/14] FIX: Update names --- mne/epochs.py | 10 +++++----- mne/io/constants.py | 8 ++++---- mne/io/tests/test_constants.py | 5 +---- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/mne/epochs.py b/mne/epochs.py index f5aae3bed20..e71c4c3d00b 100644 --- a/mne/epochs.py +++ b/mne/epochs.py @@ -133,10 +133,10 @@ def _save_split(epochs, fname, part_idx, n_parts): # undo modifications to data data /= decal[np.newaxis, :, np.newaxis] - write_string(fid, FIFF.FIFFB_MNE_EPOCHS_DROP_LOG, + write_string(fid, FIFF.FIFF_MNE_EPOCHS_DROP_LOG, json.dumps(epochs.drop_log)) - write_int(fid, FIFF.FIFFB_MNE_EPOCHS_SELECTION, + write_int(fid, FIFF.FIFF_MNE_EPOCHS_SELECTION, epochs.selection) # And now write the next file info in case epochs are split on disk @@ -2470,10 +2470,10 @@ def _read_one_epoch_file(f, tree, preload): # Constant 305 was used before v0.11 tag = read_tag(fid, pos) bmax = float(tag.data) - elif kind == FIFF.FIFFB_MNE_EPOCHS_SELECTION: + elif kind == FIFF.FIFF_MNE_EPOCHS_SELECTION: tag = read_tag(fid, pos) selection = np.array(tag.data) - elif kind == FIFF.FIFFB_MNE_EPOCHS_DROP_LOG: + elif kind == FIFF.FIFF_MNE_EPOCHS_DROP_LOG: tag = read_tag(fid, pos) drop_log = json.loads(tag.data) @@ -2513,7 +2513,7 @@ def _read_one_epoch_file(f, tree, preload): tmax = last / info['sfreq'] event_id = (dict((str(e), e) for e in np.unique(events[:, 2])) if mappings is None else mappings) - # In case epochs didn't have a FIFF.FIFFB_MNE_EPOCHS_SELECTION tag + # In case epochs didn't have a FIFF.FIFF_MNE_EPOCHS_SELECTION tag # (version < 0.8): if selection is None: selection = np.arange(len(events)) diff --git a/mne/io/constants.py b/mne/io/constants.py index eedbf4742a2..d71c1a10672 100644 --- a/mne/io/constants.py +++ b/mne/io/constants.py @@ -848,11 +848,11 @@ def __setattr__(self, attr, val): # noqa: D105 FIFF.FIFF_MNE_RT_CLIENT_ID = 3701 # realtime client # MNE epochs bookkeeping -FIFF.FIFFB_MNE_EPOCHS_SELECTION = 3800 # the epochs selection -FIFF.FIFFB_MNE_EPOCHS_DROP_LOG = 3801 # the drop log +FIFF.FIFF_MNE_EPOCHS_SELECTION = 3800 # the epochs selection +FIFF.FIFF_MNE_EPOCHS_DROP_LOG = 3801 # the drop log # MNE annotations -FIFF.FIFFB_MNE_ANNOTATIONS = 3810 # annotations +FIFF.FIFFB_MNE_ANNOTATIONS = 3810 # annotations block # MNE Metadata Dataframes -FIFF.FIFFB_MNE_METADATA = 3811 # metadata dataframes +FIFF.FIFFB_MNE_METADATA = 3811 # metadata dataframes block diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index 6546c6b352d..c63d950324b 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -139,10 +139,7 @@ def test_constants(tmpdir): elif name.startswith('FIFFB_'): # We put these in the tag section rather than the IOD/block section # -> XXX fix this by moving them to the IOD files in fiff-constants - if name in ('FIFFB_MNE_ANNOTATIONS', - 'FIFFB_MNE_EPOCHS_DROP_LOG', - 'FIFFB_MNE_EPOCHS_SELECTION', - 'FIFFB_MNE_METADATA'): + if name in 'FIFFB_MNE_ANNOTATIONS': assert val in tags continue assert val in iod, (val, name) From 5f5b85da22fecba4f6d06c6e40538377422fd2ac Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 11 Jul 2018 10:32:57 -0400 Subject: [PATCH 03/14] FIX: Tests --- mne/io/ctf/tests/test_ctf.py | 8 ++++++-- mne/io/tests/test_constants.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/mne/io/ctf/tests/test_ctf.py b/mne/io/ctf/tests/test_ctf.py index d01dee6860c..84b25a19ca5 100644 --- a/mne/io/ctf/tests/test_ctf.py +++ b/mne/io/ctf/tests/test_ctf.py @@ -105,8 +105,12 @@ def test_read_ctf(): assert_array_equal(raw.ch_names, raw_c.ch_names) assert_allclose(raw.times, raw_c.times) assert_allclose(raw._cals, raw_c._cals) - assert_equal(raw.info['meas_id']['version'], - raw_c.info['meas_id']['version']) + assert_equal(raw.info['meas_id']['usecs'], + raw_c.info['meas_id']['usecs']) + py_time = raw.info['meas_id']['secs'] + c_time = raw_c.info['meas_id']['secs'] + max_offset = 24 * 60 * 60 # probably overkill but covers timezone + assert c_time - max_offset <= py_time <= c_time for t in ('dev_head_t', 'dev_ctf_t', 'ctf_head_t'): assert_allclose(raw.info[t]['trans'], raw_c.info[t]['trans'], rtol=1e-4, atol=1e-7) diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index c63d950324b..e0a88af7226 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -139,7 +139,7 @@ def test_constants(tmpdir): elif name.startswith('FIFFB_'): # We put these in the tag section rather than the IOD/block section # -> XXX fix this by moving them to the IOD files in fiff-constants - if name in 'FIFFB_MNE_ANNOTATIONS': + if name in ('FIFFB_MNE_ANNOTATIONS', 'FIFFB_MNE_METADATA'): assert val in tags continue assert val in iod, (val, name) From 1ba5a3095cca034600e3882d3b132cc0c819ce99 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 11 Sep 2018 12:48:28 -0400 Subject: [PATCH 04/14] FIX: Fix constant checks --- mne/externals/decorator.py | 2 +- mne/io/tests/test_constants.py | 25 +++++-------------------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/mne/externals/decorator.py b/mne/externals/decorator.py index 7fe2cf1fcbf..c87eef99ab7 100644 --- a/mne/externals/decorator.py +++ b/mne/externals/decorator.py @@ -59,7 +59,7 @@ def __iter__(self): def get_init(cls): return cls.__init__.__func__ -DEF = re.compile('\s*def\s*([_\w][_\w\d]*)\s*\(') +DEF = re.compile(r'\s*def\s*([_\w][_\w\d]*)\s*\(') # basic functionality class FunctionMaker(object): diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index e0a88af7226..75d4bf21f9a 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -10,11 +10,10 @@ from mne.utils import _fetch_file, requires_good_network +# These are oddities that we won't address: iod_dups = (355, 359) # these are in both MEGIN and MNE files -tag_bads = (299,) # no proper dtype or units (sss_expansion) -# XXX -> ask MEGIN folks what it should be tag_dups = (3501, 3507) # in both MEGIN and MNE files - +# The tests should also be improved, see several XXX below. _dir_ignore_names = ('clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values') @@ -77,13 +76,8 @@ def test_constants(tmpdir): assert len(line) in (1, 2, 3), line desc = '' if len(line) == 1 else line[1] line = line[0].split() - if int(line[1]) in tag_bads: - assert len(line) == 2 - kind, id_ = line - dtype, unit = '-', '-' - else: - assert len(line) == 4, line - kind, id_, dtype, unit = line + assert len(line) == 4, line + kind, id_, dtype, unit = line id_ = int(id_) val = [kind, dtype, unit] assert id_ not in tags, (tags.get(id_), val) @@ -137,11 +131,6 @@ def test_constants(tmpdir): assert name in ('FIFFC_MAJOR_VERSION', 'FIFFC_MINOR_VERSION', 'FIFFC_VERSION') elif name.startswith('FIFFB_'): - # We put these in the tag section rather than the IOD/block section - # -> XXX fix this by moving them to the IOD files in fiff-constants - if name in ('FIFFB_MNE_ANNOTATIONS', 'FIFFB_MNE_METADATA'): - assert val in tags - continue assert val in iod, (val, name) elif name.startswith('FIFFT_'): continue # XXX add check for this @@ -153,13 +142,9 @@ def test_constants(tmpdir): continue # XXX add check for this elif name.startswith('FWD_'): # These are not FIFF constants really - # -> XXX remove from FIFF to forward.py namespace? + # XXX remove from FIFF to forward.py namespace continue elif name.startswith('FIFF_'): - if name == 'FIFF_MNE_KIT_SYSTEM_ID': - # This is missing - # -> XXX add this to Tags_MNE - continue assert val in tags, (name, val) else: unknowns.append((name, val)) From 3cf0540099e27664f515cb8d4d05db985c170621 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 11 Sep 2018 14:47:36 -0400 Subject: [PATCH 05/14] FIX: Last fixes in this round --- mne/io/ctf/tests/test_ctf.py | 8 +-- mne/io/tests/test_constants.py | 128 ++++++++++++++++++++++++++++++--- 2 files changed, 120 insertions(+), 16 deletions(-) diff --git a/mne/io/ctf/tests/test_ctf.py b/mne/io/ctf/tests/test_ctf.py index 84b25a19ca5..18d6f59de78 100644 --- a/mne/io/ctf/tests/test_ctf.py +++ b/mne/io/ctf/tests/test_ctf.py @@ -105,12 +105,8 @@ def test_read_ctf(): assert_array_equal(raw.ch_names, raw_c.ch_names) assert_allclose(raw.times, raw_c.times) assert_allclose(raw._cals, raw_c._cals) - assert_equal(raw.info['meas_id']['usecs'], - raw_c.info['meas_id']['usecs']) - py_time = raw.info['meas_id']['secs'] - c_time = raw_c.info['meas_id']['secs'] - max_offset = 24 * 60 * 60 # probably overkill but covers timezone - assert c_time - max_offset <= py_time <= c_time + assert_equal(raw.info['meas_id']['version'], + raw_c.info['meas_id']['version'] + 1) for t in ('dev_head_t', 'dev_ctf_t', 'ctf_head_t'): assert_allclose(raw.info[t]['trans'], raw_c.info[t]['trans'], rtol=1e-4, atol=1e-7) diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index 75d4bf21f9a..1ee7ede03df 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -3,6 +3,7 @@ # License: BSD (3-clause) import os.path as op +import re import shutil import zipfile @@ -18,6 +19,42 @@ _dir_ignore_names = ('clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values') +# XXX These should all probably be added to the FIFF constants +_missing_names = ( + 'FIFFV_BEM_APPROX_CONST', + 'FIFFV_BEM_APPROX_LINEAR', + 'FIFFV_COIL_ARTEMIS123_GRAD', + 'FIFFV_COIL_ARTEMIS123_REF_GRAD', + 'FIFFV_COIL_ARTEMIS123_REF_MAG', + 'FIFFV_COIL_BABY_REF_MAG', + 'FIFFV_COIL_BABY_REF_MAG2', + 'FIFFV_COIL_KRISS_GRAD', + 'FIFFV_COIL_POINT_MAGNETOMETER_X', + 'FIFFV_COIL_POINT_MAGNETOMETER_Y', + 'FIFFV_COIL_SAMPLE_TMS_PLANAR', + 'FIFFV_NEXT_SEQ', + 'FIFFV_NEXT_NONE', + 'FIFFV_PROJ_ITEM_DIP_FIX', + 'FIFFV_PROJ_ITEM_DIP_ROT', + 'FIFFV_PROJ_ITEM_FIELD', + 'FIFFV_PROJ_ITEM_HOMOG_FIELD', + 'FIFFV_PROJ_ITEM_HOMOG_GRAD', + 'FIFFV_PROJ_ITEM_NONE', + 'FIFF_UNIT_AM_M2', + 'FIFF_UNIT_AM_M3', + 'FIFFV_MNE_COORD_4D_HEAD', + 'FIFFV_MNE_COORD_CTF_DEVICE', + 'FIFFV_MNE_COORD_CTF_HEAD', + 'FIFFV_MNE_COORD_FS_TAL', + 'FIFFV_MNE_COORD_FS_TAL_GTZ', + 'FIFFV_MNE_COORD_FS_TAL_LTZ', + 'FIFFV_MNE_COORD_KIT_HEAD', + 'FIFFV_MNE_COORD_MNI_TAL', + 'FIFFV_MNE_COORD_MRI_VOXEL', + 'FIFFV_MNE_COORD_RAS', + 'FIFFV_MNE_COORD_TUFTS_EEG', +) + @requires_good_network def test_constants(tmpdir): @@ -110,6 +147,57 @@ def test_constants(tmpdir): assert id_ not in tags, (tags.get(id_), val) tags[id_] = val + # Types and enums + defines = dict() # maps the other way (name->val) + types = dict() + used_enums = ('unit', 'unitm', 'coil', 'aspect', 'bem_surf_id', + 'ch_type', 'coord', 'mri_pixel', 'point', 'role', + 'hand', 'sex', + 'mne_cov_ch', 'mne_ori', 'mne_map', 'covariance_type', + 'mne_priors', 'mne_space', 'mne_surf') + enums = dict((k, dict()) for k in used_enums) + in_ = None + re_prim = re.compile(r'^primitive\((.*)\)\s*(\S*)\s*"(.*)"$') + re_enum = re.compile(r'^enum\((\S*)\)\s*".*"$') + re_enum_entry = re.compile(r'\s*(\S*)\s*(\S*)\s*"(.*)"$') + re_defi = re.compile(r'#define\s*(\S*)\s*(\S*)\s*"(.*)"$') + for extra in ('', '_MNE'): + with open(op.join(tmpdir, 'DictionaryTypes%s.txt' + % (extra,)), 'rb') as fid: + for li, line in enumerate(fid): + line = line.decode('ISO-8859-1').strip() + if in_ is None: + p = re_prim.match(line) + e = re_enum.match(line) + d = re_defi.match(line) + if p is not None: + t, s, d = p.groups() + s = int(s) + assert s not in types + types[s] = [t, d] + elif e is not None: + # entering an enum + this_enum = e.group(1) + if this_enum in enums: + in_ = enums[e.group(1)] + elif d is not None: + t, s, d = d.groups() + s = int(s) + defines[t] = [s, d] + else: + assert not line.startswith('enum(') + else: # in an enum + if line == '{': + continue + elif line == '}': + in_ = None + continue + t, s, d = re_enum_entry.match(line).groups() + s = int(s) + if t != 'ecg' and s != 3: # ecg defined the same way + assert s not in in_ + in_[s] = [t, d] + # # Assertions # @@ -123,26 +211,46 @@ def test_constants(tmpdir): # Assert that all our constants are in the dict # (we are not necessarily complete the other way) for name in sorted(dir(FIFF)): - if name.startswith('_') or name in _dir_ignore_names: + if name.startswith('_') or name in _dir_ignore_names or \ + name in _missing_names: continue val = getattr(FIFF, name) - if name.startswith('FIFFC_'): + if name in defines: + assert defines[name][0] == val + elif name.startswith('FIFFC_'): # Checked above assert name in ('FIFFC_MAJOR_VERSION', 'FIFFC_MINOR_VERSION', 'FIFFC_VERSION') elif name.startswith('FIFFB_'): assert val in iod, (val, name) elif name.startswith('FIFFT_'): - continue # XXX add check for this + assert val in types, (val, name) elif name.startswith('FIFFV_'): - continue # XXX add check for this - elif name.startswith('FIFF_UNIT_'): - continue # XXX add check for this - elif name.startswith('FIFF_UNITM_'): # multipliers - continue # XXX add check for this + if name.startswith('FIFFV_MNE_') and name.endswith('_ORI'): + this_enum = 'mne_ori' + elif name.startswith('FIFFV_MNE_') and name.endswith('_COV'): + this_enum = 'covariance_type' + elif name.startswith('FIFFV_MNE_COORD'): + this_enum = 'coord' # weird wrapper + elif name.endswith('_CH') or '_QUAT_' in name or name in \ + ('FIFFV_DIPOLE_WAVE', 'FIFFV_GOODNESS_FIT', + 'FIFFV_HPI_ERR', 'FIFFV_HPI_G', 'FIFFV_HPI_MOV'): + this_enum = 'ch_type' + elif name.startswith('FIFFV_SUBJ_'): + this_enum = name.split('_')[2].lower() + else: + for this_enum in used_enums: + if name.startswith('FIFFV_' + this_enum.upper()): + break + else: + raise RuntimeError('Could not find %s' % (name,)) + assert this_enum in used_enums, name + assert val in enums[this_enum], (val, name) + elif name.startswith('FIFF_UNIT'): # units and multipliers + this_enum = name.split('_')[1].lower() + assert val in enums[this_enum], (name, val) elif name.startswith('FWD_'): - # These are not FIFF constants really - # XXX remove from FIFF to forward.py namespace + # XXX remove from FIFF to forward.py namespace at some point? continue elif name.startswith('FIFF_'): assert val in tags, (name, val) From f52fd7241b872560d4ae3e3282097bec2fa87ba6 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 11 Sep 2018 15:47:36 -0400 Subject: [PATCH 06/14] ENH: Make new FWD namespace --- mne/bem.py | 10 +++---- mne/forward/_compute_forward.py | 4 +-- mne/forward/_field_interpolation.py | 7 ++--- mne/forward/_make_forward.py | 10 +++---- mne/io/constants.py | 45 +++++++++++++++-------------- mne/io/tests/test_constants.py | 4 --- mne/preprocessing/maxwell.py | 7 ++--- 7 files changed, 42 insertions(+), 45 deletions(-) diff --git a/mne/bem.py b/mne/bem.py index c272754d404..7cac38fac09 100644 --- a/mne/bem.py +++ b/mne/bem.py @@ -15,8 +15,7 @@ import numpy as np from scipy import linalg -from .transforms import _ensure_trans, apply_trans -from .io.constants import FIFF +from .io.constants import FIFF, FWD from .io.write import (start_file, start_block, write_float, write_int, write_float_matrix, write_int_matrix, end_block, end_file) @@ -26,6 +25,7 @@ from .surface import (read_surface, write_surface, complete_surface_info, _compute_nearest, _get_ico_surface, read_tri, _fast_cross_nd_sum, _get_solids) +from .transforms import _ensure_trans, apply_trans from .utils import (verbose, logger, run_subprocess, get_subjects_dir, warn, _pl, _validate_type) from .fixes import einsum @@ -267,7 +267,7 @@ def _fwd_bem_linear_collocation_solution(m): m['solution'] = _fwd_bem_multi_solution(coeff, m['gamma'], nps) if len(m['surfs']) == 3: ip_mult = m['sigma'][1] / m['sigma'][2] - if ip_mult <= FIFF.FWD_BEM_IP_APPROACH_LIMIT: + if ip_mult <= FWD.BEM_IP_APPROACH_LIMIT: logger.info('IP approach required...') logger.info(' Matrix coefficients (homog)...') coeff = _fwd_bem_lin_pot_coeff([m['surfs'][-1]]) @@ -278,7 +278,7 @@ def _fwd_bem_linear_collocation_solution(m): 'IP approach...') _fwd_bem_ip_modify_solution(m['solution'], ip_solution, ip_mult, nps) - m['bem_method'] = FIFF.FWD_BEM_LINEAR_COLL + m['bem_method'] = FWD.BEM_LINEAR_COLL logger.info("Solution ready.") @@ -1521,7 +1521,7 @@ def write_bem_solution(fname, bem): _write_bem_surfaces_block(fid, bem['surfs']) # The potential solution if 'solution' in bem: - if bem['bem_method'] != FIFF.FWD_BEM_LINEAR_COLL: + if bem['bem_method'] != FWD.BEM_LINEAR_COLL: raise RuntimeError('Only linear collocation supported') write_int(fid, FIFF.FIFF_BEM_APPROX, FIFF.FIFFV_BEM_APPROX_LINEAR) write_float_matrix(fid, FIFF.FIFF_BEM_POT_SOLUTION, diff --git a/mne/forward/_compute_forward.py b/mne/forward/_compute_forward.py index 8c18e580abf..91977439c33 100644 --- a/mne/forward/_compute_forward.py +++ b/mne/forward/_compute_forward.py @@ -17,7 +17,7 @@ from copy import deepcopy from ..surface import fast_cross_3d, _project_onto_surface -from ..io.constants import FIFF +from ..io.constants import FIFF, FWD from ..transforms import apply_trans from ..utils import logger, verbose, _pl from ..parallel import parallel_func @@ -682,7 +682,7 @@ def _prep_field_computation(rr, bem, fwd_data, n_jobs, verbose=None): """ bem_rr = mults = mri_Q = head_mri_t = None if not bem['is_sphere']: - if bem['bem_method'] != FIFF.FWD_BEM_LINEAR_COLL: + if bem['bem_method'] != FWD.BEM_LINEAR_COLL: raise RuntimeError('only linear collocation supported') # Store (and apply soon) μ_0/(4π) factor before source computations mults = np.repeat(bem['source_mult'] / (4.0 * np.pi), diff --git a/mne/forward/_field_interpolation.py b/mne/forward/_field_interpolation.py index 4d3aab78695..356569b8dad 100644 --- a/mne/forward/_field_interpolation.py +++ b/mne/forward/_field_interpolation.py @@ -5,8 +5,8 @@ import numpy as np from scipy import linalg +from ..io.constants import FWD from ..bem import _check_origin -from ..io.constants import FIFF from ..io.pick import pick_types, pick_info from ..surface import get_head_surf, get_meg_helmet_surf @@ -23,9 +23,8 @@ def _is_axial_coil(coil): """Determine if the coil is axial.""" - is_ax = coil['coil_class'] in (FIFF.FWD_COILC_MAG, - FIFF.FWD_COILC_AXIAL_GRAD, - FIFF.FWD_COILC_AXIAL_GRAD2) + is_ax = coil['coil_class'] in ( + FWD.COILC_MAG, FWD.COILC_AXIAL_GRAD, FWD.COILC_AXIAL_GRAD2) return is_ax diff --git a/mne/forward/_make_forward.py b/mne/forward/_make_forward.py index 9493ff6c465..4b8bff5e36d 100644 --- a/mne/forward/_make_forward.py +++ b/mne/forward/_make_forward.py @@ -10,9 +10,10 @@ from os import path as op import numpy as np +from ._compute_forward import _compute_forwards from ..io import read_info, _loc_to_coil_trans, _loc_to_eeg_loc, Info from ..io.pick import _has_kit_refs, pick_types, pick_info -from ..io.constants import FIFF +from ..io.constants import FIFF, FWD from ..transforms import (_ensure_trans, transform_surface_to, apply_trans, _get_trans, _print_coord_trans, _coord_frame_name, Transform) @@ -26,11 +27,10 @@ from ..externals.six import string_types from .forward import Forward, _merge_meg_eeg_fwds, convert_forward_solution -from ._compute_forward import _compute_forwards -_accuracy_dict = dict(normal=FIFF.FWD_COIL_ACCURACY_NORMAL, - accurate=FIFF.FWD_COIL_ACCURACY_ACCURATE) +_accuracy_dict = dict(normal=FWD.COIL_ACCURACY_NORMAL, + accurate=FWD.COIL_ACCURACY_ACCURATE) @verbose @@ -180,7 +180,7 @@ def _create_eeg_el(ch, t=None): # The electrode location cosmag = r0ex.copy() _normalize_vectors(cosmag) - res = dict(chname=ch['ch_name'], coil_class=FIFF.FWD_COILC_EEG, w=w, + res = dict(chname=ch['ch_name'], coil_class=FWD.COILC_EEG, w=w, accuracy=_accuracy_dict['normal'], type=ch['coil_type'], coord_frame=t['to'], rmag=r0ex, cosmag=cosmag) return res diff --git a/mne/io/constants.py b/mne/io/constants.py index d71c1a10672..585bbab9e3e 100644 --- a/mne/io/constants.py +++ b/mne/io/constants.py @@ -646,27 +646,30 @@ def __setattr__(self, attr, val): # noqa: D105 # # FWD Types # -FIFF.FWD_COIL_UNKNOWN = 0 -FIFF.FWD_COILC_UNKNOWN = 0 -FIFF.FWD_COILC_EEG = 1000 -FIFF.FWD_COILC_MAG = 1 -FIFF.FWD_COILC_AXIAL_GRAD = 2 -FIFF.FWD_COILC_PLANAR_GRAD = 3 -FIFF.FWD_COILC_AXIAL_GRAD2 = 4 - -FIFF.FWD_COIL_ACCURACY_POINT = 0 -FIFF.FWD_COIL_ACCURACY_NORMAL = 1 -FIFF.FWD_COIL_ACCURACY_ACCURATE = 2 - -FIFF.FWD_BEM_UNKNOWN = -1 -FIFF.FWD_BEM_CONSTANT_COLL = 1 -FIFF.FWD_BEM_LINEAR_COLL = 2 - -FIFF.FWD_BEM_IP_APPROACH_LIMIT = 0.1 - -FIFF.FWD_BEM_LIN_FIELD_SIMPLE = 1 -FIFF.FWD_BEM_LIN_FIELD_FERGUSON = 2 -FIFF.FWD_BEM_LIN_FIELD_URANKAR = 3 + +FWD = BunchConst() + +FWD.COIL_UNKNOWN = 0 +FWD.COILC_UNKNOWN = 0 +FWD.COILC_EEG = 1000 +FWD.COILC_MAG = 1 +FWD.COILC_AXIAL_GRAD = 2 +FWD.COILC_PLANAR_GRAD = 3 +FWD.COILC_AXIAL_GRAD2 = 4 + +FWD.COIL_ACCURACY_POINT = 0 +FWD.COIL_ACCURACY_NORMAL = 1 +FWD.COIL_ACCURACY_ACCURATE = 2 + +FWD.BEM_UNKNOWN = -1 +FWD.BEM_CONSTANT_COLL = 1 +FWD.BEM_LINEAR_COLL = 2 + +FWD.BEM_IP_APPROACH_LIMIT = 0.1 + +FWD.BEM_LIN_FIELD_SIMPLE = 1 +FWD.BEM_LIN_FIELD_FERGUSON = 2 +FWD.BEM_LIN_FIELD_URANKAR = 3 # # Data types diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index 1ee7ede03df..ab51e144630 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -14,7 +14,6 @@ # These are oddities that we won't address: iod_dups = (355, 359) # these are in both MEGIN and MNE files tag_dups = (3501, 3507) # in both MEGIN and MNE files -# The tests should also be improved, see several XXX below. _dir_ignore_names = ('clear', 'copy', 'fromkeys', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values') @@ -249,9 +248,6 @@ def test_constants(tmpdir): elif name.startswith('FIFF_UNIT'): # units and multipliers this_enum = name.split('_')[1].lower() assert val in enums[this_enum], (name, val) - elif name.startswith('FWD_'): - # XXX remove from FIFF to forward.py namespace at some point? - continue elif name.startswith('FIFF_'): assert val in tags, (name, val) else: diff --git a/mne/preprocessing/maxwell.py b/mne/preprocessing/maxwell.py index 0e6e2f0457c..1013edb6063 100644 --- a/mne/preprocessing/maxwell.py +++ b/mne/preprocessing/maxwell.py @@ -22,7 +22,7 @@ _sh_complex_to_real, _sh_real_to_complex, _sh_negate) from ..forward import _concatenate_coils, _prep_meg_channels, _create_meg_coils from ..surface import _normalize_vectors -from ..io.constants import FIFF +from ..io.constants import FIFF, FWD from ..io.meas_info import _simplify_info from ..io.proc_history import _read_ctc from ..io.write import _generate_meas_id, DATE_NONE @@ -748,7 +748,7 @@ def _copy_preload_add_channels(raw, add_channels): dict(ch_name='CHPI%03d' % (ii + 1), logno=ii + 1, scanno=off + ii + 1, unit_mul=-1, range=1., unit=-1, kind=kinds[ii], coord_frame=FIFF.FIFFV_COORD_UNKNOWN, - cal=1e-4, coil_type=FIFF.FWD_COIL_UNKNOWN, loc=np.zeros(12)) + cal=1e-4, coil_type=FWD.COIL_UNKNOWN, loc=np.zeros(12)) for ii in range(len(kinds))] raw.info['chs'].extend(chpi_chs) raw.info._update_redundant() @@ -1002,8 +1002,7 @@ def _concatenate_sph_coils(coils): def _get_mag_mask(coils): """Get the coil_scale for Maxwell filtering.""" - return np.array([coil['coil_class'] == FIFF.FWD_COILC_MAG - for coil in coils]) + return np.array([coil['coil_class'] == FWD.COILC_MAG for coil in coils]) def _sss_basis_basic(exp, coils, mag_scale=100., method='standard'): From b92f1c9673a7d1c4753f41f0e05b8518c473534f Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Tue, 11 Sep 2018 16:43:51 -0400 Subject: [PATCH 07/14] FIX: Use codeload --- mne/io/tests/test_constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index ab51e144630..d7b70694a55 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -59,8 +59,8 @@ def test_constants(tmpdir): """Test compensation.""" dest = op.join(tmpdir, 'fiff.zip') - _fetch_file('https://api.github.com/repos/mne-tools/fiff-constants' - '/zipball/master', dest) + _fetch_file('https://codeload.github.com/mne-tools/fiff-constants/zip/' + 'master', dest) names = list() with zipfile.ZipFile(dest, 'r') as ff: for name in ff.namelist(): From 44f011d72aab397a082190c629e3d1c812b04359 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 10:27:32 -0400 Subject: [PATCH 08/14] FIX: Better redirect following --- mne/utils.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/mne/utils.py b/mne/utils.py index 6cd8e02fc6c..ee91e48a418 100644 --- a/mne/utils.py +++ b/mne/utils.py @@ -2021,16 +2021,20 @@ def _fetch_file(url, file_name, print_destination=True, resume=True, verbose_bool = (logger.level <= 20) # 20 is info try: # Check file size and displaying it alongside the download url - u = urllib.request.urlopen(url, timeout=timeout) - u.close() - # this is necessary to follow any redirects - url = u.geturl() - u = urllib.request.urlopen(url, timeout=timeout) - try: - file_size = int(u.headers.get('Content-Length', '1').strip()) - finally: - u.close() - del u + # this loop is necessary to follow any redirects + for _ in range(10): # 10 really should be sufficient... + u = urllib.request.urlopen(url, timeout=timeout) + try: + last_url, url = url, u.geturl() + if url == last_url: + file_size = int( + u.headers.get('Content-Length', '1').strip()) + break + finally: + u.close() + del u + else: + raise RuntimeError('Too many redirects') logger.info('Downloading %s (%s)' % (url, sizeof_fmt(file_size))) # Triage resume From 637dbcbbb254ae736c80053ac96b43a0eddddf18 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 10:44:51 -0400 Subject: [PATCH 09/14] FIX: String encoding --- mne/io/tag.py | 4 ++-- mne/io/tests/test_constants.py | 14 +++----------- mne/io/write.py | 2 +- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/mne/io/tag.py b/mne/io/tag.py index 3521548d945..973318768a4 100644 --- a/mne/io/tag.py +++ b/mne/io/tag.py @@ -341,9 +341,9 @@ def _read_simple(fid, tag, shape, rlims, dtype): def _read_string(fid, tag, shape, rlims): """Read a string tag.""" - # Always decode to unicode. + # Always decode to ISO 8859-1 (FIFF standard). d = _frombuffer_rows(fid, tag.size, dtype='>c', shape=shape, rlims=rlims) - return text_type(d.tostring().decode('utf-8', 'ignore')) + return text_type(d.tostring().decode('ISO 8859-1', 'ignore')) def _read_complex_float(fid, tag, shape, rlims): diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index d7b70694a55..fb93d6ebbc7 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -20,8 +20,8 @@ # XXX These should all probably be added to the FIFF constants _missing_names = ( - 'FIFFV_BEM_APPROX_CONST', - 'FIFFV_BEM_APPROX_LINEAR', + 'FIFFV_NEXT_SEQ', + 'FIFFV_NEXT_NONE', 'FIFFV_COIL_ARTEMIS123_GRAD', 'FIFFV_COIL_ARTEMIS123_REF_GRAD', 'FIFFV_COIL_ARTEMIS123_REF_MAG', @@ -31,14 +31,6 @@ 'FIFFV_COIL_POINT_MAGNETOMETER_X', 'FIFFV_COIL_POINT_MAGNETOMETER_Y', 'FIFFV_COIL_SAMPLE_TMS_PLANAR', - 'FIFFV_NEXT_SEQ', - 'FIFFV_NEXT_NONE', - 'FIFFV_PROJ_ITEM_DIP_FIX', - 'FIFFV_PROJ_ITEM_DIP_ROT', - 'FIFFV_PROJ_ITEM_FIELD', - 'FIFFV_PROJ_ITEM_HOMOG_FIELD', - 'FIFFV_PROJ_ITEM_HOMOG_GRAD', - 'FIFFV_PROJ_ITEM_NONE', 'FIFF_UNIT_AM_M2', 'FIFF_UNIT_AM_M3', 'FIFFV_MNE_COORD_4D_HEAD', @@ -151,7 +143,7 @@ def test_constants(tmpdir): types = dict() used_enums = ('unit', 'unitm', 'coil', 'aspect', 'bem_surf_id', 'ch_type', 'coord', 'mri_pixel', 'point', 'role', - 'hand', 'sex', + 'hand', 'sex', 'proj_item', 'bem_approx', 'mne_cov_ch', 'mne_ori', 'mne_map', 'covariance_type', 'mne_priors', 'mne_space', 'mne_surf') enums = dict((k, dict()) for k in used_enums) diff --git a/mne/io/write.py b/mne/io/write.py index b57978dcc04..1f9016b5480 100644 --- a/mne/io/write.py +++ b/mne/io/write.py @@ -116,7 +116,7 @@ def write_julian(fid, kind, data): def write_string(fid, kind, data): """Write a string tag.""" - str_data = data.encode('utf-8') # Use unicode or bytes depending on Py2/3 + str_data = data.encode('ISO 8859-1') data_size = len(str_data) # therefore compute size here my_dtype = '>a' # py2/3 compatible on writing -- don't ask me why if data_size > 0: From 26ad2a8874e1d824bf829e67e019aa5dffbab4e3 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 10:48:37 -0400 Subject: [PATCH 10/14] FIX: Better alias --- mne/io/tag.py | 4 ++-- mne/io/tests/test_constants.py | 2 +- mne/io/write.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/mne/io/tag.py b/mne/io/tag.py index 973318768a4..284507199dd 100644 --- a/mne/io/tag.py +++ b/mne/io/tag.py @@ -341,9 +341,9 @@ def _read_simple(fid, tag, shape, rlims, dtype): def _read_string(fid, tag, shape, rlims): """Read a string tag.""" - # Always decode to ISO 8859-1 (FIFF standard). + # Always decode to ISO 8859-1 / latin1 (FIFF standard). d = _frombuffer_rows(fid, tag.size, dtype='>c', shape=shape, rlims=rlims) - return text_type(d.tostring().decode('ISO 8859-1', 'ignore')) + return text_type(d.tostring().decode('latin1', 'ignore')) def _read_complex_float(fid, tag, shape, rlims): diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index fb93d6ebbc7..f90951b302d 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -71,7 +71,7 @@ def test_constants(tmpdir): for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']: with open(op.join(tmpdir, name), 'rb') as fid: for line in fid: - line = line.decode('ISO-8859-1').strip() + line = line.decode('latin1').strip() if line.startswith('# Packing revision'): assert fiff_version is None fiff_version = line.split()[-1] diff --git a/mne/io/write.py b/mne/io/write.py index 1f9016b5480..0bd7e112782 100644 --- a/mne/io/write.py +++ b/mne/io/write.py @@ -116,7 +116,7 @@ def write_julian(fid, kind, data): def write_string(fid, kind, data): """Write a string tag.""" - str_data = data.encode('ISO 8859-1') + str_data = data.encode('latin1') data_size = len(str_data) # therefore compute size here my_dtype = '>a' # py2/3 compatible on writing -- don't ask me why if data_size > 0: From 6b18f44d69d300c0267f9006d522fe3c598dd002 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 11:29:01 -0400 Subject: [PATCH 11/14] FIX: Clearer unicode test --- mne/io/fiff/tests/test_raw_fiff.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mne/io/fiff/tests/test_raw_fiff.py b/mne/io/fiff/tests/test_raw_fiff.py index 683896aba6f..cff3768cdce 100644 --- a/mne/io/fiff/tests/test_raw_fiff.py +++ b/mne/io/fiff/tests/test_raw_fiff.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # Author: Alexandre Gramfort # Denis Engemann # @@ -513,16 +514,16 @@ def test_io_raw(): rng = np.random.RandomState(0) tempdir = _TempDir() # test unicode io - for chars in [b'\xc3\xa4\xc3\xb6\xc3\xa9', b'a']: + for chars in [u'äöé', 'a']: with read_raw_fif(fif_fname) as r: assert ('Raw' in repr(r)) assert (op.basename(fif_fname) in repr(r)) - desc1 = r.info['description'] = chars.decode('utf-8') + r.info['description'] = chars temp_file = op.join(tempdir, 'raw.fif') r.save(temp_file, overwrite=True) with read_raw_fif(temp_file) as r2: desc2 = r2.info['description'] - assert_equal(desc1, desc2) + assert desc2 == chars # Let's construct a simple test for IO first raw = read_raw_fif(fif_fname).crop(0, 3.5) From 70e3cbe36e76e5671428721671f68990ed0937b5 Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 12:51:14 -0400 Subject: [PATCH 12/14] FIX: Ancient pytest --- mne/io/tests/test_constants.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index f90951b302d..5f54368800c 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -50,7 +50,7 @@ @requires_good_network def test_constants(tmpdir): """Test compensation.""" - dest = op.join(tmpdir, 'fiff.zip') + dest = tmpdir.join('fiff.zip') _fetch_file('https://codeload.github.com/mne-tools/fiff-constants/zip/' 'master', dest) names = list() @@ -59,7 +59,7 @@ def test_constants(tmpdir): if 'Dictionary' in name: ff.extract(name, tmpdir) names.append(op.basename(name)) - shutil.move(op.join(tmpdir, name), op.join(tmpdir, names[-1])) + shutil.move(tmpdir.join(name), tmpdir.join(names[-1])) names = sorted(names) assert names == ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt', 'DictionaryStructures.txt', @@ -69,7 +69,7 @@ def test_constants(tmpdir): iod = dict() fiff_version = None for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']: - with open(op.join(tmpdir, name), 'rb') as fid: + with open(tmpdir.join(name), 'rb') as fid: for line in fid: line = line.decode('latin1').strip() if line.startswith('# Packing revision'): @@ -94,7 +94,7 @@ def test_constants(tmpdir): iod[id_] = [kind, desc] # Tags (MEGIN) tags = dict() - with open(op.join(tmpdir, 'DictionaryTags.txt'), 'rb') as fid: + with open(tmpdir.join('DictionaryTags.txt'), 'rb') as fid: for line in fid: line = line.decode('ISO-8859-1').strip() if (line.startswith('#') or line.startswith('alias') or @@ -111,7 +111,7 @@ def test_constants(tmpdir): assert id_ not in tags, (tags.get(id_), val) tags[id_] = val # Tags (MNE) - with open(op.join(tmpdir, 'DictionaryTags_MNE.txt'), 'rb') as fid: + with open(tmpdir.join('DictionaryTags_MNE.txt'), 'rb') as fid: for li, line in enumerate(fid): line = line.decode('ISO-8859-1').strip() # ignore continuation lines (*) @@ -153,8 +153,8 @@ def test_constants(tmpdir): re_enum_entry = re.compile(r'\s*(\S*)\s*(\S*)\s*"(.*)"$') re_defi = re.compile(r'#define\s*(\S*)\s*(\S*)\s*"(.*)"$') for extra in ('', '_MNE'): - with open(op.join(tmpdir, 'DictionaryTypes%s.txt' - % (extra,)), 'rb') as fid: + with open(tmpdir.join( + 'DictionaryTypes%s.txt' % (extra,)), 'rb') as fid: for li, line in enumerate(fid): line = line.decode('ISO-8859-1').strip() if in_ is None: From 3565f8ffe541e35f33b5c6c984b5a1dd19d95fbc Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 14:42:08 -0400 Subject: [PATCH 13/14] FIX: Old Pytest --- mne/io/tests/test_constants.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index 5f54368800c..0f3601e6684 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -50,7 +50,8 @@ @requires_good_network def test_constants(tmpdir): """Test compensation.""" - dest = tmpdir.join('fiff.zip') + tmpdir = str(tmpdir) # old pytest... + dest = op.join(tmpdir, 'fiff.zip') _fetch_file('https://codeload.github.com/mne-tools/fiff-constants/zip/' 'master', dest) names = list() @@ -59,7 +60,7 @@ def test_constants(tmpdir): if 'Dictionary' in name: ff.extract(name, tmpdir) names.append(op.basename(name)) - shutil.move(tmpdir.join(name), tmpdir.join(names[-1])) + shutil.move(op.join(tmpdir, name), op.join(tmpdir, names[-1])) names = sorted(names) assert names == ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt', 'DictionaryStructures.txt', @@ -69,7 +70,7 @@ def test_constants(tmpdir): iod = dict() fiff_version = None for name in ['DictionaryIOD.txt', 'DictionaryIOD_MNE.txt']: - with open(tmpdir.join(name), 'rb') as fid: + with open(op.join(tmpdir, name), 'rb') as fid: for line in fid: line = line.decode('latin1').strip() if line.startswith('# Packing revision'): @@ -94,7 +95,7 @@ def test_constants(tmpdir): iod[id_] = [kind, desc] # Tags (MEGIN) tags = dict() - with open(tmpdir.join('DictionaryTags.txt'), 'rb') as fid: + with open(op.join(tmpdir, 'DictionaryTags.txt'), 'rb') as fid: for line in fid: line = line.decode('ISO-8859-1').strip() if (line.startswith('#') or line.startswith('alias') or @@ -111,7 +112,7 @@ def test_constants(tmpdir): assert id_ not in tags, (tags.get(id_), val) tags[id_] = val # Tags (MNE) - with open(tmpdir.join('DictionaryTags_MNE.txt'), 'rb') as fid: + with open(op.join(tmpdir, 'DictionaryTags_MNE.txt'), 'rb') as fid: for li, line in enumerate(fid): line = line.decode('ISO-8859-1').strip() # ignore continuation lines (*) @@ -153,8 +154,8 @@ def test_constants(tmpdir): re_enum_entry = re.compile(r'\s*(\S*)\s*(\S*)\s*"(.*)"$') re_defi = re.compile(r'#define\s*(\S*)\s*(\S*)\s*"(.*)"$') for extra in ('', '_MNE'): - with open(tmpdir.join( - 'DictionaryTypes%s.txt' % (extra,)), 'rb') as fid: + with open(op.join(tmpdir, 'DictionaryTypes%s.txt' + % (extra,)), 'rb') as fid: for li, line in enumerate(fid): line = line.decode('ISO-8859-1').strip() if in_ is None: From 3c068782709e4d34acfce92e837aa7876e9ba4ad Mon Sep 17 00:00:00 2001 From: Eric Larson Date: Wed, 12 Sep 2018 20:21:34 -0400 Subject: [PATCH 14/14] DOC: whats_new.rst --- doc/whats_new.rst | 2 ++ mne/io/tests/test_constants.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/whats_new.rst b/doc/whats_new.rst index 17b716e1cf6..b1a78214439 100644 --- a/doc/whats_new.rst +++ b/doc/whats_new.rst @@ -54,6 +54,8 @@ Bug - Fix bug of not showing ERD's in baseline rescaled tfr topomaps if grads are combined by `Erkka Heinila`_ +- Fix bug with FIF I/O where strings were written in UTF-8 format instead of Latin-1 by `Eric Larson`_ + - Fix bug with reading measurement dates from BrainVision files by `Stefan Appelhoff`_ - Fix bug with `mne.fit_dipole` where the residual was returned as ``ndarray`` instead of :class:`mne.Evoked` instance, by `Eric Larson`_ diff --git a/mne/io/tests/test_constants.py b/mne/io/tests/test_constants.py index 0f3601e6684..f746a884244 100644 --- a/mne/io/tests/test_constants.py +++ b/mne/io/tests/test_constants.py @@ -16,7 +16,10 @@ tag_dups = (3501, 3507) # in both MEGIN and MNE files _dir_ignore_names = ('clear', 'copy', 'fromkeys', 'get', 'items', 'keys', - 'pop', 'popitem', 'setdefault', 'update', 'values') + 'pop', 'popitem', 'setdefault', 'update', 'values', + 'has_key', 'iteritems', 'iterkeys', 'itervalues', # Py2 + 'viewitems', 'viewkeys', 'viewvalues', # Py2 + ) # XXX These should all probably be added to the FIFF constants _missing_names = (