From ac1f6d609231e81972581d2ca333af355d9a8086 Mon Sep 17 00:00:00 2001 From: haoxingz Date: Sat, 25 Jan 2014 11:43:50 -0800 Subject: [PATCH 1/9] WIP. Added test fistures for matlab magic. --- pymatbridge/tests/test_magic.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 pymatbridge/tests/test_magic.py diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py new file mode 100644 index 0000000..73dc0f1 --- /dev/null +++ b/pymatbridge/tests/test_magic.py @@ -0,0 +1,20 @@ +import pymatbridge as pymat +import IPython + +import random as rd +import numpy as np +import numpy.testing as npt + +class TestMagic: + + # Create an IPython shell and load Matlab magic + @classmethod + def setup_class(cls): + cls.ip = IPython.InteractiveShell() + pymat.load_ipython_extension(cls.ip) + + # Unload the magic, shut down Matlab + @classmethod + def teardown_class(cls): + pymat.unload_ipython_extension(cls.ip) + From 33dd7be77bd7841e49a6b215f90741aac81a1eda Mon Sep 17 00:00:00 2001 From: haoxingz Date: Sat, 25 Jan 2014 15:19:33 -0800 Subject: [PATCH 2/9] Added test cases for cell magic --- pymatbridge/tests/test_magic.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 73dc0f1..9980c8b 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -1,8 +1,6 @@ import pymatbridge as pymat import IPython -import random as rd -import numpy as np import numpy.testing as npt class TestMagic: @@ -11,6 +9,8 @@ class TestMagic: @classmethod def setup_class(cls): cls.ip = IPython.InteractiveShell() + cls.ip.run_cell('import random') + cls.ip.run_cell('import numpy as np') pymat.load_ipython_extension(cls.ip) # Unload the magic, shut down Matlab @@ -18,3 +18,26 @@ def setup_class(cls): def teardown_class(cls): pymat.unload_ipython_extension(cls.ip) + + # Test single operation on differnt data structures + def test_cell_magic_number(self): + # A double precision real number + self.ip.run_cell("a = np.float64(random.random())") + self.ip.run_cell_magic('matlab', '-i a -o b', 'b = a*2;') + npt.assert_almost_equal(self.ip.user_ns['b'], self.ip.user_ns['a']*2, decimal=7) + + # A complex number + self.ip.run_cell("x = 3.34+4.56j") + self.ip.run_cell_magic('matlab', '-i x -o y', 'y = x*(11.35 - 23.098j)') + self.ip.run_cell("res = x*(11.35 - 23.098j)") + npt.assert_almost_equal(self.ip.user_ns['y'], self.ip.user_ns['res'], decimal=7) + + # Random array multiplication + self.ip.run_cell("val1 = np.random.random_sample((3,3))") + self.ip.run_cell("val2 = np.random.random_sample((3,3))") + self.ip.run_cell("respy = np.dot(val1, val2)") + self.ip.run_cell_magic('matlab', '-i val1,val2 -o resmat', 'resmat = val1 * val2') + npt.assert_almost_equal(self.ip.user_ns['resmat'], self.ip.user_ns['respy'], decimal=7) + + + From 6aff98ff6b1803d4b78bd30beb5e0c65896166d5 Mon Sep 17 00:00:00 2001 From: haoxingz Date: Sun, 26 Jan 2014 14:03:41 -0800 Subject: [PATCH 3/9] Transposed the matrix after multiplication --- pymatbridge/tests/test_magic.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 9980c8b..f275970 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -32,10 +32,12 @@ def test_cell_magic_number(self): self.ip.run_cell("res = x*(11.35 - 23.098j)") npt.assert_almost_equal(self.ip.user_ns['y'], self.ip.user_ns['res'], decimal=7) + + def test_cell_magic_array(self): # Random array multiplication self.ip.run_cell("val1 = np.random.random_sample((3,3))") self.ip.run_cell("val2 = np.random.random_sample((3,3))") - self.ip.run_cell("respy = np.dot(val1, val2)") + self.ip.run_cell("respy = np.transpose(np.dot(val1, val2))") self.ip.run_cell_magic('matlab', '-i val1,val2 -o resmat', 'resmat = val1 * val2') npt.assert_almost_equal(self.ip.user_ns['resmat'], self.ip.user_ns['respy'], decimal=7) From 79600da76b5f201a4f7ec3344365ca781c3551be Mon Sep 17 00:00:00 2001 From: haoxingz Date: Sun, 26 Jan 2014 15:40:18 -0800 Subject: [PATCH 4/9] Ready to be merged. Added tests for line magic and figure --- pymatbridge/tests/test_magic.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index f275970..39fd3ae 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -42,4 +42,16 @@ def test_cell_magic_array(self): npt.assert_almost_equal(self.ip.user_ns['resmat'], self.ip.user_ns['respy'], decimal=7) - + def test_line_magic(self): + # Some operation in Matlab + self.ip.run_line_magic('matlab', 'a = [1 2 3]') + self.ip.run_line_magic('matlab', 'res = a*2') + # Get the result back to Python + self.ip.run_cell_magic('matlab', '-o actual', 'actual = res') + + self.ip.run_cell("expected = np.array([[2], [4], [6]])") + npt.assert_almost_equal(self.ip.user_ns['actual'], self.ip.user_ns['expected'], decimal=7) + + def test_figure(self): + # Just make a plot to get more testing coverage + self.ip.run_line_magic('matlab', 'plot([1 2 3])') From 36813f3b5960a079dddd07cb861674df10d09b74 Mon Sep 17 00:00:00 2001 From: arokem Date: Sun, 26 Jan 2014 21:03:26 -0800 Subject: [PATCH 5/9] BF: Make inputs and outputs from the magic conform in shape. --- pymatbridge/matlab_magic.py | 2 +- pymatbridge/tests/test_magic.py | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/pymatbridge/matlab_magic.py b/pymatbridge/matlab_magic.py index badae16..5b3336b 100644 --- a/pymatbridge/matlab_magic.py +++ b/pymatbridge/matlab_magic.py @@ -70,7 +70,7 @@ def loadmat(fname): if len(data.dtype) > 0: # must be complex data data = data['real'] + 1j * data['imag'] - return data + return np.squeeze(data.T) def matlab_converter(matlab, key): diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 39fd3ae..1369d76 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -24,22 +24,26 @@ def test_cell_magic_number(self): # A double precision real number self.ip.run_cell("a = np.float64(random.random())") self.ip.run_cell_magic('matlab', '-i a -o b', 'b = a*2;') - npt.assert_almost_equal(self.ip.user_ns['b'], self.ip.user_ns['a']*2, decimal=7) + npt.assert_almost_equal(self.ip.user_ns['b'], + self.ip.user_ns['a']*2, decimal=7) # A complex number self.ip.run_cell("x = 3.34+4.56j") self.ip.run_cell_magic('matlab', '-i x -o y', 'y = x*(11.35 - 23.098j)') self.ip.run_cell("res = x*(11.35 - 23.098j)") - npt.assert_almost_equal(self.ip.user_ns['y'], self.ip.user_ns['res'], decimal=7) + npt.assert_almost_equal(self.ip.user_ns['y'], + self.ip.user_ns['res'], decimal=7) def test_cell_magic_array(self): # Random array multiplication self.ip.run_cell("val1 = np.random.random_sample((3,3))") self.ip.run_cell("val2 = np.random.random_sample((3,3))") - self.ip.run_cell("respy = np.transpose(np.dot(val1, val2))") - self.ip.run_cell_magic('matlab', '-i val1,val2 -o resmat', 'resmat = val1 * val2') - npt.assert_almost_equal(self.ip.user_ns['resmat'], self.ip.user_ns['respy'], decimal=7) + self.ip.run_cell("respy = np.dot(val1, val2)") + self.ip.run_cell_magic('matlab', '-i val1,val2 -o resmat', + 'resmat = val1 * val2') + npt.assert_almost_equal(self.ip.user_ns['resmat'], + self.ip.user_ns['respy'], decimal=7) def test_line_magic(self): @@ -49,9 +53,18 @@ def test_line_magic(self): # Get the result back to Python self.ip.run_cell_magic('matlab', '-o actual', 'actual = res') - self.ip.run_cell("expected = np.array([[2], [4], [6]])") - npt.assert_almost_equal(self.ip.user_ns['actual'], self.ip.user_ns['expected'], decimal=7) + self.ip.run_cell("expected = np.array([2, 4, 6])") + npt.assert_almost_equal(self.ip.user_ns['actual'], + self.ip.user_ns['expected'], decimal=7) def test_figure(self): # Just make a plot to get more testing coverage self.ip.run_line_magic('matlab', 'plot([1 2 3])') + + def test_matrix(self): + self.ip.run_cell("in_array = np.array([[1,2,3], [4,5,6]])") + self.ip.run_cell_magic('matlab', '-i in_array -o out_array', + 'out_array = in_array;') + npt.assert_almost_equal(self.ip.user_ns['out_array'], + self.ip.user_ns['in_array'], + decimal=7) From e0fc70204ce01415d09826fb282adc94c2b62a8f Mon Sep 17 00:00:00 2001 From: haoxingz Date: Thu, 30 Jan 2014 12:16:18 -0800 Subject: [PATCH 6/9] 1. Added test case for Matlab struct data type 2. Partially fixed matlab_magic to handle Matlab struct(Still buggy with strings) --- pymatbridge/matlab_magic.py | 72 ++++++++++++++++++++------------- pymatbridge/tests/test_magic.py | 13 ++++++ 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/pymatbridge/matlab_magic.py b/pymatbridge/matlab_magic.py index 5b3336b..93197b4 100644 --- a/pymatbridge/matlab_magic.py +++ b/pymatbridge/matlab_magic.py @@ -28,7 +28,7 @@ has_io = False no_io_str = "Must have h5py and scipy.io to perform i/o" no_io_str += "operations with the Matlab session" - + from IPython.core.displaypub import publish_display_data from IPython.core.magic import (Magics, magics_class, cell_magic, line_magic, line_cell_magic, needs_local_scope) @@ -39,7 +39,7 @@ import pymatbridge as pymat - + class MatlabInterperterError(RuntimeError): """ Some error occurs while matlab is running @@ -52,7 +52,7 @@ def __unicode__(self): s = "Failed to parse and evaluate line %r.\n Matlab error message: %r"%\ (self.line, self.err) return s - + if PY3: __str__ = __unicode__ else: @@ -66,25 +66,41 @@ def loadmat(fname): """ f = h5py.File(fname) - data = f.values()[0][:] - if len(data.dtype) > 0: - # must be complex data - data = data['real'] + 1j * data['imag'] - return np.squeeze(data.T) + + for var_name in f.iterkeys(): + if isinstance(f[var_name], h5py.Dataset): + # Currently only supports numerical array + data = f[var_name].value + if len(data.dtype) > 0: + # must be complex data + data = data['real'] + 1j * data['imag'] + return np.squeeze(data.T) + + elif isinstance(f[var_name], h5py.Group): + data = {} + for mem_name in f[var_name].iterkeys(): + if isinstance(f[var_name][mem_name], h5py.Dataset): + data[mem_name] = f[var_name][mem_name].value + data[mem_name] = np.squeeze(data[mem_name].T) + else: + # Currently doesn't support nested struct + pass + + return data def matlab_converter(matlab, key): """ Reach into the matlab namespace and get me the value of the variable - + """ tempdir = tempfile.gettempdir() # We save as hdf5 in the matlab session, so that we can grab large # variables: matlab.run_code("save('%s/%s.mat','%s','-v7.3')"%(tempdir, key, key), maxtime=matlab.maxtime) - + return loadmat('%s/%s.mat'%(tempdir, key)) @@ -113,17 +129,17 @@ def __init__(self, shell, maxtime : float The maximal time to wait for responses for matlab (in seconds). Default: 10 seconds. - + pyconverter : callable To be called on matlab variables returning into the ipython namespace - + matlab_converter : callable - To be called on values in ipython namespace before + To be called on values in ipython namespace before assigning to variables in matlab. cache_display_data : bool - If True, the published results of the final call to R are + If True, the published results of the final call to R are cached in the variable 'display_cache'. """ @@ -133,7 +149,7 @@ def __init__(self, shell, self.Matlab = pymat.Matlab(matlab, maxtime=maxtime) self.Matlab.start() self.pyconverter = pyconverter - self.matlab_converter = matlab_converter + self.matlab_converter = matlab_converter def __del__(self): """shut down the Matlab server when the object dies. @@ -154,9 +170,9 @@ def eval(self, line): if run_dict['success'] == 'false': raise MatlabInterperterError(line, run_dict['content']['stdout']) - # This is the matlab stdout: + # This is the matlab stdout: return run_dict - + @magic_arguments() @argument( '-i', '--input', action='append', @@ -180,7 +196,7 @@ def matlab(self, line, cell=None, local_ns=None): """ Execute code in matlab - + """ args = parse_argstring(self.matlab, line) @@ -210,7 +226,7 @@ def matlab(self, line, cell=None, local_ns=None): except KeyError: val = self.shell.user_ns[input] # We save these input arguments into a .mat file: - tempdir = tempfile.gettempdir() + tempdir = tempfile.gettempdir() sio.savemat('%s/%s.mat'%(tempdir, input), eval("dict(%s=val)"%input), oned_as='row') @@ -219,7 +235,7 @@ def matlab(self, line, cell=None, local_ns=None): else: raise RuntimeError(no_io_str) - + text_output = '' #imgfiles = [] @@ -234,14 +250,14 @@ def matlab(self, line, cell=None, local_ns=None): e_s += "\n-----------------------" e_s += "\nAre you sure Matlab is started?" raise RuntimeError(e_s) - - + + text_output += result_dict['content']['stdout'] # Figures get saved by matlab in reverse order... imgfiles = result_dict['content']['figures'][::-1] data_dir = result_dict['content']['datadir'] - + display_data = [] if text_output: display_data.append(('MatlabMagic.matlab', @@ -251,7 +267,7 @@ def matlab(self, line, cell=None, local_ns=None): if len(imgf): # Store the path to the directory so that you can delete it # later on: - image = open(imgf, 'rb').read() + image = open(imgf, 'rb').read() display_data.append(('MatlabMagic.matlab', {'image/png': image})) @@ -261,7 +277,7 @@ def matlab(self, line, cell=None, local_ns=None): # Delete the temporary data files created by matlab: if len(data_dir): rmtree(data_dir) - + if args.output: if has_io: for output in ','.join(args.output).split(','): @@ -269,8 +285,8 @@ def matlab(self, line, cell=None, local_ns=None): output)}) else: raise RuntimeError(no_io_str) - - + + _loaded = False def load_ipython_extension(ip, **kwargs): """Load the extension in IPython.""" @@ -278,7 +294,7 @@ def load_ipython_extension(ip, **kwargs): if not _loaded: ip.register_magics(MatlabMagics(ip, **kwargs)) _loaded = True - + def unload_ipython_extension(ip): global _loaded if _loaded: diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 1369d76..6bde54f 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -61,6 +61,7 @@ def test_figure(self): # Just make a plot to get more testing coverage self.ip.run_line_magic('matlab', 'plot([1 2 3])') + def test_matrix(self): self.ip.run_cell("in_array = np.array([[1,2,3], [4,5,6]])") self.ip.run_cell_magic('matlab', '-i in_array -o out_array', @@ -68,3 +69,15 @@ def test_matrix(self): npt.assert_almost_equal(self.ip.user_ns['out_array'], self.ip.user_ns['in_array'], decimal=7) + + # Matlab struct type should be converted to a Python dict + def test_struct(self): + self.ip.run_cell('num = 2.567') + self.ip.run_cell('num_array = np.array([1.2,3.4,5.6])') + self.ip.run_cell('str = "Hello World"') + self.ip.run_cell_magic('matlab', '-i num,num_array,str -o obj', + 'obj.num = num;obj.num_array = num_array;obj.str = str;') + npt.assert_equal(isinstance(self.ip.user_ns['obj'], dict), True) + npt.assert_equal(self.ip.user_ns['obj']['num'], self.ip.user_ns['num']) + npt.assert_equal(self.ip.user_ns['obj']['num_array'], self.ip.user_ns['num_array']) + npt.assert_equal(self.ip.user_ns['obj']['str'], self.ip.user_ns['str']) From 8746cb8a84f7773af8990a09d7a2ad960dda0591 Mon Sep 17 00:00:00 2001 From: haoxingz Date: Thu, 30 Jan 2014 20:59:04 -0800 Subject: [PATCH 7/9] Quick hack for the string bug --- pymatbridge/matlab_magic.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pymatbridge/matlab_magic.py b/pymatbridge/matlab_magic.py index 93197b4..0b1205c 100644 --- a/pymatbridge/matlab_magic.py +++ b/pymatbridge/matlab_magic.py @@ -82,6 +82,13 @@ def loadmat(fname): if isinstance(f[var_name][mem_name], h5py.Dataset): data[mem_name] = f[var_name][mem_name].value data[mem_name] = np.squeeze(data[mem_name].T) + # This is a quick hack. Need to find a better way + # to identify string + if data[mem_name].dtype == 'uint16': + result = '' + for asc in data[mem_name]: + result += chr(asc) + data[mem_name] = result else: # Currently doesn't support nested struct pass From 634ee88a88ae69d8fc70f24ab7049673988d98da Mon Sep 17 00:00:00 2001 From: haoxingz Date: Thu, 30 Jan 2014 21:01:58 -0800 Subject: [PATCH 8/9] Fixed typo --- pymatbridge/tests/test_magic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/tests/test_magic.py b/pymatbridge/tests/test_magic.py index 6bde54f..47987bf 100644 --- a/pymatbridge/tests/test_magic.py +++ b/pymatbridge/tests/test_magic.py @@ -19,7 +19,7 @@ def teardown_class(cls): pymat.unload_ipython_extension(cls.ip) - # Test single operation on differnt data structures + # Test single operation on different data structures def test_cell_magic_number(self): # A double precision real number self.ip.run_cell("a = np.float64(random.random())") From 2e3bfd9cf93fcbbff642c775342f81aec1119a23 Mon Sep 17 00:00:00 2001 From: haoxingz Date: Mon, 3 Feb 2014 11:00:09 -0800 Subject: [PATCH 9/9] Fixed string handling by checking the attribute of the dataset --- pymatbridge/matlab_magic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pymatbridge/matlab_magic.py b/pymatbridge/matlab_magic.py index 0b1205c..59a21e1 100644 --- a/pymatbridge/matlab_magic.py +++ b/pymatbridge/matlab_magic.py @@ -80,11 +80,17 @@ def loadmat(fname): data = {} for mem_name in f[var_name].iterkeys(): if isinstance(f[var_name][mem_name], h5py.Dataset): + # Check if the dataset is a string + attr = h5py.AttributeManager(f[var_name][mem_name]) + if (attr.__getitem__('MATLAB_class') == 'char'): + is_string = True + else: + is_string = False + data[mem_name] = f[var_name][mem_name].value data[mem_name] = np.squeeze(data[mem_name].T) - # This is a quick hack. Need to find a better way - # to identify string - if data[mem_name].dtype == 'uint16': + + if is_string: result = '' for asc in data[mem_name]: result += chr(asc)