From a94e8a9094e5d7b5cf78a030772a104227fbed53 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Mon, 7 Apr 2014 19:12:30 +1000 Subject: [PATCH 01/33] Initial pythonic API teaser --- pymatbridge/matlab/matlabserver.m | 72 ++++--- pymatbridge/matlab/util/pymat_call.m | 56 +++++ pymatbridge/matlab/util/pymat_eval.m | 18 +- pymatbridge/matlab/util/pymat_feval.m | 16 +- pymatbridge/matlab/util/pymat_get_variable.m | 29 +-- pymatbridge/pymatbridge.py | 214 ++++++++++++------- 6 files changed, 263 insertions(+), 142 deletions(-) create mode 100644 pymatbridge/matlab/util/pymat_call.m diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 55b223b..2f5d298 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -6,36 +6,46 @@ function matlabserver(socket_address) json.startup messenger('init', socket_address); -while(1) - msg_in = messenger('listen'); - req = json.load(msg_in); - - switch(req.cmd) - case {'connect'} - messenger('respond', 'connected'); - - case {'exit'} - messenger('exit'); - clear mex; - break; - - case {'run_function'} - fhandle = str2func('pymat_feval'); - resp = feval(fhandle, req); - messenger('respond', resp); - - case {'run_code'} - fhandle = str2func('pymat_eval'); - resp = feval(fhandle, req); - messenger('respond', resp); - - case {'get_var'} - fhandle = str2func('pymat_get_variable'); - resp = feval(fhandle, req); - messenger('respond', resp); - - otherwise - messenger('respond', 'i dont know what you want'); +while true + % don't let any errors escape (and crash the server) + try + msg_in = messenger('listen'); + req = json.load(msg_in); + + switch(req.cmd) + case {'connect'} + messenger('respond', 'connected'); + + case {'exit'} + messenger('exit'); + clear mex; + break; + + case {'call'} + fhandle = str2func('pymat_call') + resp = feval(fhandle, req) + messenger('respond', resp) + + case {'eval'} + fhandle = str2func('pymat_eval'); + resp = feval(fhandle, req); + messenger('respond', resp); + + case {'get_var'} + fhandle = str2func('pymat_get_variable'); + resp = feval(fhandle, req); + messenger('respond', resp); + + otherwise + messenger('respond', 'unrecognized command'); + end + + catch exception + response.success = false; + response.result = exception.identifier; + response.message = exception.message; + + json_response = json.dump(response); + messenger('respond', 'json_response'); end - end diff --git a/pymatbridge/matlab/util/pymat_call.m b/pymatbridge/matlab/util/pymat_call.m new file mode 100644 index 0000000..6bdd418 --- /dev/null +++ b/pymatbridge/matlab/util/pymat_call.m @@ -0,0 +1,56 @@ +function json_response = pymat_call(req) + + if ~isfield(req, 'func') + response.message = 'No function given as func POST parameter'; + json_response = json.dump(response); + return + end + + if ~isfield(req, 'args') + req.args = {} + end + + func = str2func(req.func); + if isfield(req, 'nout') + nout = req.nout + else + try + nout = min(abs(nargout(func)), 1); + catch + nout = 0; + end + end + + try + switch nout + case 0 + feval(func, req.args{:}); + response.result = true; + case 1 + a = feval(func, req.args{:}); + response.result = a; + case 2 + [a, b] = feval(func, req.args{:}); + response.result = {a, b}; + case 3 + [a, b, c] = feval(func, req.args{:}); + response.result = {a, b, c}; + case 4 + [a, b, c, d] = feval(func, req.args{:}); + response.result = {a, b, c, d}; + default + % varargout or throw exception + response.result = feval(func, req.args{:}); + end + response.success = true; + response.message = 'Successfully completed request'; + catch exception + response.success = false; + response.result = exception.identifier; + response.message = exception.message; + end + + json_response = json.dump(response); + return + +end diff --git a/pymatbridge/matlab/util/pymat_eval.m b/pymatbridge/matlab/util/pymat_eval.m index b071cb8..8846d82 100644 --- a/pymatbridge/matlab/util/pymat_eval.m +++ b/pymatbridge/matlab/util/pymat_eval.m @@ -6,7 +6,7 @@ % % This allows you to run any matlab code. To be used with webserver.m. % HTTP POST to /web_eval.m with the following parameters: -% code: a string which contains the code to be run in the matlab session +% expr: a string which contains the expression to evaluate in the matlab session % % Should return a json object containing the result % @@ -17,26 +17,26 @@ response.content = ''; -code_check = false; +expr_check = false; if size(field_names) - if isfield(req, 'code') - code_check = true; + if isfield(req, 'expr') + expr_check = true; end end -if ~code_check - response.message = 'No code provided as POST parameter'; +if ~expr_check + response.message = 'No expression provided as POST parameter'; json_response = json.dump(response); return; end -code = req.code; +expr = req.expr; try % tempname is less likely to get bonked by another process. diary_file = [tempname() '_diary.txt']; diary(diary_file); - evalin('base', code); + evalin('base', expr); diary('off'); datadir = fullfile(tempdir(),'MatlabData'); @@ -69,7 +69,7 @@ response.content.stdout = ME.message; end -response.content.code = code; +response.content.expr = expr; json_response = json.dump(response); diff --git a/pymatbridge/matlab/util/pymat_feval.m b/pymatbridge/matlab/util/pymat_feval.m index 2fd3a48..ccd7bf4 100644 --- a/pymatbridge/matlab/util/pymat_feval.m +++ b/pymatbridge/matlab/util/pymat_feval.m @@ -1,7 +1,19 @@ -% Max Jaderberg 2011 - function json_response = matlab_feval(req) + if ~isfield(req, 'func_path') + response.success = false; + response.message = 'No function given as func_path POST parameter'; + json_response = json.dump(response); + end + + if ~isfield(req, 'func_args') + req.func_args = {} + end + + [dir, func_name, ext] = fileparts(req.func_path); + try + response.result = + response.success = 'false'; field_names = fieldnames(req); diff --git a/pymatbridge/matlab/util/pymat_get_variable.m b/pymatbridge/matlab/util/pymat_get_variable.m index c37fa90..b96956c 100644 --- a/pymatbridge/matlab/util/pymat_get_variable.m +++ b/pymatbridge/matlab/util/pymat_get_variable.m @@ -2,31 +2,16 @@ % Reach into the current namespace get a variable in json format that can % be returned as part of a response -response.success = 'false'; - -field_names = fieldnames(req); - -response.content = ''; - -varname_check = false; -if size(field_names) - if isfield(req, 'varname') - varname_check = true; + try + response.result = evalin('base', req.varname); + response.success = true; + catch exception + response.success = false; + response.result = exception.identifier; + response.message = exception.message; end -end -if ~varname_check - response.message = 'No variable name provided as input argument'; json_response = json.dump(response); return -end - - -varname = req.varname; - -response.var = evalin('base', varname); - -json_response = json.dump(response); -return end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 9e9a347..82657d5 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -8,16 +8,40 @@ This is a modified version using ZMQ, Haoxing Zhang Jan.2014 """ - -import numpy as np -import os, time -import zmq -import subprocess +from __future__ import print_function +import collections +import functools +import json +import os import platform +import subprocess import sys +import time +import types +import weakref +import zmq +try: + basestring + DEVNULL = open(os.devnull, 'w') +except: + basestring = str + DEVNULL = subprocess.DEVNULL + +# ---------------------------------------------------------------------------- +# HELPERS +# ---------------------------------------------------------------------------- +def chain(*iterables): + for iterable in iterables: + if not isinstance(iterable, collections.Iterable) or isinstance(iterable, basestring): + yield iterable + else: + for item in iterable: + yield item -import json +# ---------------------------------------------------------------------------- +# JSON EXTENSION +# ---------------------------------------------------------------------------- # JSON encoder extension to handle complex numbers class ComplexEncoder(json.JSONEncoder): def default(self, obj): @@ -33,32 +57,18 @@ def as_complex(dct): return dct +# ---------------------------------------------------------------------------- +# MATLAB +# ---------------------------------------------------------------------------- MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) -# Start a Matlab server and bind it to a ZMQ socket(TCP/IPC) -def _run_matlab_server(matlab_bin, matlab_socket_addr, matlab_log, matlab_id, matlab_startup_options): - command = matlab_bin - command += ' %s ' % matlab_startup_options - command += ' -r "' - command += "addpath(genpath(" - command += "'%s'" % MATLAB_FOLDER - command += ')), matlabserver(\'%s\'),exit"' % matlab_socket_addr - - if matlab_log: - command += ' -logfile ./pymatbridge/logs/matlablog_%s.txt > ./pymatbridge/logs/bashlog_%s.txt' % (matlab_id, matlab_id) - - subprocess.Popen(command, shell = True, stdin=subprocess.PIPE) - - return True - - class Matlab(object): """ A class for communicating with a matlab session """ def __init__(self, matlab='matlab', socket_addr=None, - id='python-matlab-bridge', log=False, maxtime=60, + id='python-matlab-bridge', log=False, maxtime=30, platform=None, startup_options=None): """ Initialize this thing. @@ -94,6 +104,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.running = False self.matlab = matlab self.socket_addr = socket_addr + self.blacklist = set() self.id = id self.log = log @@ -114,44 +125,63 @@ def __init__(self, matlab='matlab', socket_addr=None, else: self.startup_options = ' -nodesktop -nodisplay' - self.context = None - self.socket = None + self.context = zmq.Context() + self.socket = self.context.socket(zmq.REQ) + # generate some useful matlab builtins + getattr(self, 'addpath') + getattr(self, 'eval') + getattr(self, 'help') + getattr(self, 'license') + getattr(self, 'run') + getattr(self, 'run') + getattr(self, 'version') + + def __del__(self): + self.socket.close() + try: + self.matlab_process.terminate() + except: + pass + + # ------------------------------------------------------------------------ + # START/STOP SERVER + # ------------------------------------------------------------------------ # Start server/client session and make the connection def start(self): # Start the MATLAB server in a new process - print "Starting MATLAB on ZMQ socket %s" % (self.socket_addr) - print "Send 'exit' command to kill the server" - _run_matlab_server(self.matlab, self.socket_addr, self.log, self.id, self.startup_options) + command = chain( + self.matlab, + self.startup_options, + '-r', + "\"addpath(genpath('%s')),matlabserver('%s'),exit\"" % (MATLAB_FOLDER, self.socket_addr) + ) + + command = ' '.join(command) + print('Starting Matlab subprocess', end='') + self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) # Start the client - self.context = zmq.Context() - self.socket = self.context.socket(zmq.REQ) self.socket.connect(self.socket_addr) - self.started = True # Test if connection is established if (self.is_connected()): - print "MATLAB started and connected!" - return True + print('ready') else: - print "MATLAB failed to start" - return False + raise RuntimeError('Matlab failed to start') # Stop the Matlab server def stop(self): - req = json.dumps(dict(cmd="exit"), cls=ComplexEncoder) + req = json.dumps({'cmd': 'exit'}, cls=ComplexEncoder) self.socket.send(req) resp = self.socket.recv_string() # Matlab should respond with "exit" if successful if resp == "exit": - print "MATLAB closed" - + print("Matlab subprocess stopped") self.started = False - return True # To test if the client can talk to the server def is_connected(self): @@ -171,10 +201,11 @@ def is_connected(self): else: return False except zmq.ZMQError: - np.disp(".", linefeed=False) + print('.', end='') + sys.stdout.flush() time.sleep(1) if (time.time() - start_time > self.maxtime) : - print "Matlab session timed out after %d seconds" % (self.maxtime) + print('failed to connect to Matlab after %d seconds' % self.maxtime) return False @@ -185,30 +216,7 @@ def is_function_processor_working(self): else: return False - - # Run a function in Matlab and return the result - def run_func(self, func_path, func_args=None, maxtime=None): - if self.running: - time.sleep(0.05) - - req = dict(cmd="run_function") - req['func_path'] = func_path - req['func_args'] = func_args - - req = json.dumps(req, cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) - - return resp - - # Run some code in Matlab command line provide by a string - def run_code(self, code, maxtime=None): - if self.running: - time.sleep(0.05) - - req = dict(cmd="run_code") - req['code'] = code + def _execute(self, req): req = json.dumps(req, cls=ComplexEncoder) self.socket.send(req) resp = self.socket.recv_string() @@ -216,16 +224,66 @@ def run_code(self, code, maxtime=None): return resp - def get_variable(self, varname, maxtime=None): - if self.running: - time.sleep(0.05) - - req = dict(cmd="get_var") - req['varname'] = varname - req = json.dumps(req, cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) - - return resp['var'] + # ------------------------------------------------------------------------ + # PYTHONIC API + # ------------------------------------------------------------------------ + def __getattr__(self, name): + if name in self.blacklist: + raise AttributeError(attribute_msg.format(name)) + method_instance = Method(self, name) + method_instance.__name__ = name + ' (unverified)' + setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + return getattr(self, name) + + def get_variable(self, varname, timeout=None): + req = { + 'cmd': 'get_var', + 'varname': varname, + } + resp = self._execute(req) + if not resp['success']: + raise RuntimeError(resp['result'] +': '+ resp['message']) + return resp['result'] + + def clear_blacklist(self): + self.blacklist = set() + + +# ---------------------------------------------------------------------------- +# MATLAB METHOD +# ---------------------------------------------------------------------------- +attribute_msg = "attribute '{0}' does not correspond to a Matlab function and was blacklisted" + +class Method(object): + + def __init__(self, parent, name): + self.name = name + self.doc = None + + def __call__(self, parent, *args, **kwargs): + nout = kwargs.pop('nout', None) + args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) + req = { + 'cmd': 'call', + 'func': self.name, + 'args': args + } + if nout: + req['nout'] = nout + resp = parent()._execute(req) + if not resp['success']: + if resp['result'] == 'MATLAB:UndefinedFunction': + parent().blacklist.add(self.name) + delattr(parent(), self.name) + raise AttributeError(attribute_msg.format(self.name)) + raise RuntimeError(resp['result'] +': '+ resp['message']) + else: + self.__name__ = self.name + ' (verified)' + return resp['result'] + + @property + def __doc__(self, parent): + if not self.doc: + self.doc = parent().help(self.name) + return self.doc From dba311441d86c406f3ebe50d274d6da88124f67c Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Tue, 8 Apr 2014 22:49:13 +1000 Subject: [PATCH 02/33] streamlined API --- pymatbridge/matlab/matlabserver.m | 134 +++++--- pymatbridge/matlab/util/pymat_call.m | 56 ---- pymatbridge/pymatbridge.py | 483 ++++++++++++++++++--------- 3 files changed, 418 insertions(+), 255 deletions(-) delete mode 100644 pymatbridge/matlab/util/pymat_call.m diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 2f5d298..279f54d 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -1,51 +1,95 @@ function matlabserver(socket_address) -% This function takes a socket address as input and initiates a ZMQ session -% over the socket. I then enters the listen-respond mode until it gets an -% "exit" command - -json.startup -messenger('init', socket_address); - -while true - % don't let any errors escape (and crash the server) - try - msg_in = messenger('listen'); - req = json.load(msg_in); - - switch(req.cmd) - case {'connect'} - messenger('respond', 'connected'); - - case {'exit'} - messenger('exit'); - clear mex; - break; - - case {'call'} - fhandle = str2func('pymat_call') - resp = feval(fhandle, req) - messenger('respond', resp) - - case {'eval'} - fhandle = str2func('pymat_eval'); - resp = feval(fhandle, req); - messenger('respond', resp); - - case {'get_var'} - fhandle = str2func('pymat_get_variable'); - resp = feval(fhandle, req); - messenger('respond', resp); - - otherwise - messenger('respond', 'unrecognized command'); +% MATLABSERVER Run a Matlab server to handle requests from Python over ZMQ +% +% MATLABSERVER(SOCKET_ADDRESS) initiates a ZMQ session over the provided +% SOCKET_ADDRESS. Once started, it executes client requests and returns +% the response. +% +% The recognized requests are: +% 'ping': Elicit a response from the server +% 'exit': Request the server to shutdown +% 'call': Call a Matlab function with the provdided arguments + + json.startup + messenger('init', socket_address); + + while true + % don't let any errors escape (and crash the server) + try + msg_in = messenger('listen'); + req = json.load(msg_in); + + switch(req.cmd) + case {'ping'} + messenger('respond', 'pong'); + + case {'exit'} + messenger('exit'); + clear mex; + break; + + case {'call'} + resp = call(req); + json_resp = json.dump(resp); + messenger('respond', json_resp); + + otherwise + throw(MException('MATLAB:matlabserver', ['Unrecognized command ' req.cmd])) + end + + catch exception + % format the exception and pass it back to the client + resp.success = false; + resp.result = exception.identifier; + resp.message = exception.message; + + json_resp = json.dump(resp); + messenger('respond', json_resp); end + end +end + + +function resp = call(req) +% CALL Call a Matlab function +% +% RESPONSE = CALL(REQUEST) calls Matlab's FEVAL function, intelligently +% handling the number of input and output arguments so that the argument +% spec is satisfied. +% +% The REQUEST is a struct with three fields: +% 'func': The name of the function to execute +% 'args': A cell array of args to expand into the function arguments +% 'nout': The number of output arguments requested + + % function of no arguments + if ~isfield(req, 'args') + req.args = {} + end - catch exception - response.success = false; - response.result = exception.identifier; - response.message = exception.message; + % determine the number of output arguments + % TODO: What should the default behaviour be? + func = str2func(req.func); + nout = req.nout; + if isempty(nout) + try + nout = min(abs(nargout(func)), 1); + catch + nout = 1; + end + end - json_response = json.dump(response); - messenger('respond', 'json_response'); + % call the function, taking care of broadcasting outputs + switch nout + case 0 + func(req.args{:}); + case 1 + resp.result = func(req.args{:}); + otherwise + [resp.result{1:nout}] = func(req.args{:}); end + + % build the response + resp.success = true; + resp.message = 'Successfully completed request'; end diff --git a/pymatbridge/matlab/util/pymat_call.m b/pymatbridge/matlab/util/pymat_call.m deleted file mode 100644 index 6bdd418..0000000 --- a/pymatbridge/matlab/util/pymat_call.m +++ /dev/null @@ -1,56 +0,0 @@ -function json_response = pymat_call(req) - - if ~isfield(req, 'func') - response.message = 'No function given as func POST parameter'; - json_response = json.dump(response); - return - end - - if ~isfield(req, 'args') - req.args = {} - end - - func = str2func(req.func); - if isfield(req, 'nout') - nout = req.nout - else - try - nout = min(abs(nargout(func)), 1); - catch - nout = 0; - end - end - - try - switch nout - case 0 - feval(func, req.args{:}); - response.result = true; - case 1 - a = feval(func, req.args{:}); - response.result = a; - case 2 - [a, b] = feval(func, req.args{:}); - response.result = {a, b}; - case 3 - [a, b, c] = feval(func, req.args{:}); - response.result = {a, b, c}; - case 4 - [a, b, c, d] = feval(func, req.args{:}); - response.result = {a, b, c, d}; - default - % varargout or throw exception - response.result = feval(func, req.args{:}); - end - response.success = true; - response.message = 'Successfully completed request'; - catch exception - response.success = false; - response.result = exception.identifier; - response.message = exception.message; - end - - json_response = json.dump(response); - return - -end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 82657d5..c35179f 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -1,12 +1,9 @@ -""" -pymatbridge -=========== - -This is a module for communicating and running +"""Main module for running and communicating with Matlab subprocess -Part of Python-MATLAB-bridge, Max Jaderberg 2012 +pymatbridge.py provides Matlab, the main class used to communicate +with the Matlab executable. Each instance starts and manages its +own Matlab subprocess. -This is a modified version using ZMQ, Haoxing Zhang Jan.2014 """ from __future__ import print_function import collections @@ -21,16 +18,31 @@ import weakref import zmq try: - basestring - DEVNULL = open(os.devnull, 'w') + # Python 2 + basestring + DEVNULL = open(os.devnull, 'w') except: - basestring = str - DEVNULL = subprocess.DEVNULL + # Python 3 + basestring = str + DEVNULL = subprocess.DEVNULL + # ---------------------------------------------------------------------------- # HELPERS # ---------------------------------------------------------------------------- def chain(*iterables): + """Yield elements from each iterable in order + + Make an iterator that returns elements from the first iterable until + it is exhausted, then proceeds to the next iterable, until all of the + iterables are exhausted. Unlike itertools.chain, strings are not treated + as iterable, and thus not expanded into characters: + chain([1, 2, 3, 4], 'string') --> 1, 2, 3, 4, 'string' + + Returns: + generator: A generator which yields items from the iterables + + """ for iterable in iterables: if not isinstance(iterable, collections.Iterable) or isinstance(iterable, basestring): yield iterable @@ -39,19 +51,49 @@ def chain(*iterables): yield item +class AttributeDict(dict): + """A dictionary with attribute-like access + + Values within an AttributeDict can be accessed either via + d[key] or d.key. + See: http://stackoverflow.com/a/14620633 + + """ + def __init__(self, *args, **kwargs): + super(AttributeDict, self).__init__(*args, **kwargs) + self.__dict__ = self + + # ---------------------------------------------------------------------------- # JSON EXTENSION # ---------------------------------------------------------------------------- -# JSON encoder extension to handle complex numbers class ComplexEncoder(json.JSONEncoder): + """A JSON extension for encoding complex numbers + + ComplexEncoder encodes complex numbers as a mapping, + complex --> {'real': complex.real, 'imag': complex.imag} + + """ def default(self, obj): if isinstance(obj, complex): return {'real':obj.real, 'imag':obj.imag} # Handle the default case return json.JSONEncoder.default(self, obj) -# JSON decoder for complex numbers def as_complex(dct): + """A JSON extension for decoding complex numbers + + as_complex decodes mappings of the form {'real': real_val, 'imag': imag_val} + into a single value of type complex + + Args: + dct: A dictionary + + Returns: + complex: A complex number if the dictionary represents an encoding + of a complex number, else the dictionary + + """ if 'real' in dct and 'imag' in dct: return complex(dct['real'], dct['imag']) return dct @@ -60,101 +102,96 @@ def as_complex(dct): # ---------------------------------------------------------------------------- # MATLAB # ---------------------------------------------------------------------------- -MATLAB_FOLDER = '%s/matlab' % os.path.realpath(os.path.dirname(__file__)) - class Matlab(object): - """ - A class for communicating with a matlab session - """ - def __init__(self, matlab='matlab', socket_addr=None, - id='python-matlab-bridge', log=False, maxtime=30, + id='python-matlab-bridge', log=False, timeout=30, platform=None, startup_options=None): - """ - Initialize this thing. - - Parameters - ---------- - - matlab : str - A string that woul start matlab at the terminal. Per default, this - is set to 'matlab', so that you can alias in your bash setup - - socket_addr : str - A string that represents a valid ZMQ socket address, such as - "ipc:///tmp/pymatbridge", "tcp://127.0.0.1:55555", etc. - - id : str - An identifier for this instance of the pymatbridge - - log : bool - Whether to save a log file in some known location. - - maxtime : float - The maximal time to wait for a response from matlab (optional, - Default is 10 sec) - - platform : string - The OS of the machine on which this is running. Per default this - will be taken from sys.platform. + """Execute functions in a Matlab subprocess via Python + + Matlab provides a pythonic interface for accessing functions in Matlab. + It works by starting a Matlab subprocess and establishing a connection + to it using ZMQ. Function calls are serialized and executed remotely. + + Keyword Args: + matlab (str): A string to the Matlab executable. This defaults + to 'matlab', assuming the executable is on your PATH + socket_addr (str): A string the represents a valid ZMQ socket + address, such as "ipc:///tmp/pymatbridge", "tcp://127.0.0.1:5555" + id (str): An identifier for this instance of the pymatbridge + log (bool): Log status and error messages + timeout: The maximum time to wait for a response from the Matlab + process before timing out (default 30 seconds) + platform (str): The OS of the machine running Matlab. By default + this is determined automatically from sys.platform + startup_options (list): A list of switches that should be passed + to the Matlab subprocess at startup. By default, switches are + passed to disable the graphical session. For a full list of + available switches see: + Windows: http://www.mathworks.com.au/help/matlab/ref/matlabwindows.html + UNIX: http://www.mathworks.com.au/help/matlab/ref/matlabunix.html """ + self.MATLAB_FOLDER = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'matlab') + # Setup internal state variables self.started = False - self.running = False self.matlab = matlab self.socket_addr = socket_addr - self.blacklist = set() - self.id = id self.log = log - self.maxtime = maxtime + self.timeout = timeout - if platform is None: - self.platform = sys.platform + # determine the platform-specific options + self.platform = platform if platform else sys.platform + if self.platform == 'win32': + default_socket_addr = "tcp://127.0.0.1:55555" + default_options = ['-automation', '-nofigureWindows'] else: - self.platform = platform + default_socket_addr = "ipc:///tmp/pymatbridge" + default_options = ['-nodesktop', '-nodisplay'] - if self.socket_addr is None: # use the default - self.socket_addr = "tcp://127.0.0.1:55555" if self.platform == "win32" else "ipc:///tmp/pymatbridge" - - if startup_options: - self.startup_options = startup_options - elif self.platform == 'win32': - self.startup_options = ' -automation -noFigureWindows' - else: - self.startup_options = ' -nodesktop -nodisplay' + self.socket_addr = socket_addr if socket_addr else default_socket_addr + self.startup_options = startup_options if startup_options else default_options + # initialize the ZMQ socket self.context = zmq.Context() self.socket = self.context.socket(zmq.REQ) - # generate some useful matlab builtins - getattr(self, 'addpath') - getattr(self, 'eval') - getattr(self, 'help') - getattr(self, 'license') - getattr(self, 'run') - getattr(self, 'run') - getattr(self, 'version') + # auto-generate some useful matlab builtins + self.bind_method('exist', unconditionally=True) + self.bind_method('addpath', unconditionally=True) + self.bind_method('eval', unconditionally=True) + self.bind_method('help', unconditionally=True) + self.bind_method('license', unconditionally=True) + self.bind_method('run', unconditionally=True) + self.bind_method('version', unconditionally=True) def __del__(self): - self.socket.close() - try: - self.matlab_process.terminate() - except: - pass + """Forcibly cleanup resources + + The user should always call Matlab.stop to gracefully shutdown the Matlab + process before Matlab leaves scope, but in case they don't, attempt + to cleanup so we don't leave an orphaned process lying around. + + """ + self.stop() - # ------------------------------------------------------------------------ - # START/STOP SERVER - # ------------------------------------------------------------------------ - # Start server/client session and make the connection def start(self): - # Start the MATLAB server in a new process + """Start a new Matlab subprocess and attempt to connect to it via ZMQ + + Raises: + RuntimeError: If Matlab is already running, or failed to start + + """ + if self.started: + raise RuntimeError('Matlab is already running') + + # build the command command = chain( self.matlab, self.startup_options, '-r', - "\"addpath(genpath('%s')),matlabserver('%s'),exit\"" % (MATLAB_FOLDER, self.socket_addr) + "\"addpath(genpath('%s')),matlabserver('%s'),exit\"" % (self.MATLAB_FOLDER, self.socket_addr) ) command = ' '.join(command) @@ -167,123 +204,261 @@ def start(self): # Test if connection is established if (self.is_connected()): - print('ready') + print('started') else: + self.started = False raise RuntimeError('Matlab failed to start') + def stop(self, timeout=1): + """Stop the Matlab subprocess + + Attempt to gracefully shutdown the Matlab subprocess. If it fails to + stop within the timeout period, terminate it forcefully. + + Args: + timeout: Time in seconds before SIGKILL is sent + + """ + if not self.started: + return - # Stop the Matlab server - def stop(self): req = json.dumps({'cmd': 'exit'}, cls=ComplexEncoder) - self.socket.send(req) - resp = self.socket.recv_string() + try: + # the user might be stopping Matlab because the socket is in a bad state + self.socket.send(req) + except: + pass - # Matlab should respond with "exit" if successful - if resp == "exit": - print("Matlab subprocess stopped") + start_time = time.time() + while time.time() - start_time < timeout: + if self.matlab_process.poll() is not None: + break + time.sleep(0.1) + else: + self.matlab_process.kill() + + # finalize + self.socket.close() self.started = False - # To test if the client can talk to the server + def restart(self): + """Restart the Matlab subprocess if the state becomes bad + + Aliases the following command: + >> matlab.stop() + >> matlab.start() + + """ + self.stop() + self.start() + def is_connected(self): + """Test if the client can talk to the server + + Raises: + RuntimeError: If there is no running Matlab subprocess to connect to + + Returns: + bool: True if the Matlab subprocess ZMQ server can be contacted, + False, otherwise + + """ if not self.started: - time.sleep(2) - return False + raise RuntimeError('No running Matlab subprocess to connect to') - req = json.dumps(dict(cmd="connect"), cls=ComplexEncoder) + req = json.dumps({'cmd': 'ping'}, cls=ComplexEncoder) self.socket.send(req) start_time = time.time() - while(True): + while time.time() - start_time < self.timeout: + time.sleep(0.5) try: - resp = self.socket.recv_string(flags=zmq.NOBLOCK) - if resp == "connected": - return True - else: - return False + self.socket.recv_string(flags=zmq.NOBLOCK) + return True except zmq.ZMQError: print('.', end='') sys.stdout.flush() - time.sleep(1) - if (time.time() - start_time > self.maxtime) : - print('failed to connect to Matlab after %d seconds' % self.maxtime) - return False + print('failed to connect to Matlab after %d seconds' % self.timeout) + return False def is_function_processor_working(self): - result = self.run_func('%s/test_functions/test_sum.m' % MATLAB_FOLDER, {'echo': 'Matlab: Function processor is working!'}) - if result['success'] == 'true': + """Check whether the Matlab subprocess can evaluate functions + + First check whether the Python client can talk to the Matlab + server, then if the server is in a state where it can successfully + evaluate a function + + Raises: + RuntimeError: If there is no running Matlab subprocess to connect to + + Returns: + bool: True if Matlab can evaluate a function, False otherwise + + """ + if not self.is_connected(): + return False + + try: + self.abs(2435) return True - else: + except: return False - def _execute(self, req): + def execute_in_matlab(self, req): + """Execute a request in the Matlab subprocess + + Args: + req (dict): A dictionary containing the request to evaluate in + Matlab. The request should contain the 'cmd' key, and the + corresponding command recognized by matlabserver.m, as well + as any arguments required by that command + + Returns: + resp (dict): A dictionary containing the response from the Matlab + server containing the keys 'success', 'result', and 'message' + + Raises: + RuntimeError: If the 'success' field of the resp object is False, + then an exception is raised with the value of the 'result' + field which will contain the identifier of the exception + raised in Matlab, and the 'message' field, which will contain + the reason for the exception + + """ + # send the request req = json.dumps(req, cls=ComplexEncoder) self.socket.send(req) - resp = self.socket.recv_string() - resp = json.loads(resp, object_hook=as_complex) + # receive the response + resp = self.socket.recv_string() + resp = AttributeDict(json.loads(resp, object_hook=as_complex)) + if not resp.success: + raise RuntimeError(resp.result +': '+ resp.message) return resp - - # ------------------------------------------------------------------------ - # PYTHONIC API - # ------------------------------------------------------------------------ def __getattr__(self, name): - if name in self.blacklist: - raise AttributeError(attribute_msg.format(name)) - method_instance = Method(self, name) - method_instance.__name__ = name + ' (unverified)' - setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) - return getattr(self, name) + return self.bind_method(name) + def bind_method(self, name, unconditionally=False): + """Generate a Matlab function and bind it to the instance + + This is where the magic happens. When an unknown attribute of the + Matlab class is requested, it is assumed to be a call to a + Matlab function, and is generated and bound to the instance. + + This works because getattr() falls back to __getattr__ only if no + attributes of the requested name can be found through normal + routes (__getattribute__, __dict__, class tree). - def get_variable(self, varname, timeout=None): - req = { - 'cmd': 'get_var', - 'varname': varname, - } - resp = self._execute(req) - if not resp['success']: - raise RuntimeError(resp['result'] +': '+ resp['message']) - return resp['result'] + bind_method first checks whether the requested name is a callable + Matlab function before generating a binding. - def clear_blacklist(self): - self.blacklist = set() + Args: + name (str): The name of the Matlab function to call + e.g. 'sqrt', 'sum', 'svd', etc + unconditionally (bool): Bind the method without performing + checks. Used to bootstrap methods that are required and + know to exist + + Returns: + Method: a reference to a newly bound Method instance if the + requested name is determined to be a callable function + + Raises: + AttributeError: if the requested name is not a callable + Matlab function + + """ + # TODO: This does not work if the function is a mex function inside a folder of the same name + if not unconditionally and not self.exist(name): + raise AttributeError("'Matlab' object has no attribute '%s'" % name) + + # create a new method instance + method_instance = Method(weakref.ref(self), name) + method_instance.__name__ = name + + # bind to the Matlab instance with a weakref (to avoid circular references) + setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + return getattr(self, name) # ---------------------------------------------------------------------------- # MATLAB METHOD # ---------------------------------------------------------------------------- -attribute_msg = "attribute '{0}' does not correspond to a Matlab function and was blacklisted" - class Method(object): def __init__(self, parent, name): + """An object representing a Matlab function + + Methods are dynamically bound to instances of Matlab objects and + represent a callable function in the Matlab subprocess. + + Args: + parent: A reference to the parent (Matlab instance) to which the + Method is being bound + name: The name of the Matlab function this represents + + """ self.name = name + self._parent = parent self.doc = None - def __call__(self, parent, *args, **kwargs): + def __call__(self, unused_parent_weakref, *args, **kwargs): + """Call a function with the supplied arguments in the Matlab subprocess + + Args: + The *args parameter is unpacked and forwarded verbatim to Matlab. + It contains arguments in the order that they would appear in a + native function call. + + Keyword Args: + Keyword arguments are passed to Matlab in the form [key, val] so + that matlab.plot(x, y, '--', LineWidth=2) would be translated into + plot(x, y, '--', 'LineWidth', 2) + + nout (int): The number of arguments to output. By default this is + 1 for functions that return 1 or more values, and 0 for + functions that return no values. This is useful for functions + that change their behvaiour depending on the number of inputs: + U, S, V = matlab.svd(A, nout=3) + + """ + # parse out number of output arguments nout = kwargs.pop('nout', None) + + # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) - req = { - 'cmd': 'call', - 'func': self.name, - 'args': args - } - if nout: - req['nout'] = nout - resp = parent()._execute(req) - if not resp['success']: - if resp['result'] == 'MATLAB:UndefinedFunction': - parent().blacklist.add(self.name) - delattr(parent(), self.name) - raise AttributeError(attribute_msg.format(self.name)) - raise RuntimeError(resp['result'] +': '+ resp['message']) - else: - self.__name__ = self.name + ' (verified)' - return resp['result'] + + # build request + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout} + resp = self.parent.execute_in_matlab(req) + + # return the result + return resp.get('result', None) @property - def __doc__(self, parent): - if not self.doc: - self.doc = parent().help(self.name) + def parent(self): + """Get the actual parent from the stored weakref + + The parent (Matlab instance) is stored as a weak reference + to eliminate circular references from dynamically binding Methods + to Matlab. + + """ + parent = self._parent() + if parent is None: + raise AttributeError('Stale reference to attribute of non-existent Matlab object') + return parent + + @property + def __doc__(self): + """Fetch the docstring from Matlab + + Get the documentation for a Matlab function by calling Matlab's builtin + help() then returning it as the Python docstring. The result is cached + so Matlab is only ever polled on the first request + + """ + if self.doc is None: + self.doc = self.parent.help(self.name) return self.doc From 30024e95ec983af35776507c4ef6ec29cc79dc1f Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Wed, 9 Apr 2014 16:08:28 +1000 Subject: [PATCH 03/33] Added 2D numpy (de-)serialization support. 3D+ currently being mangled by Matlab --- pymatbridge/pymatbridge.py | 86 ++++++++++++++++++++++++++++---------- 1 file changed, 63 insertions(+), 23 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index c35179f..b0b77c7 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -9,6 +9,7 @@ import collections import functools import json +import numpy as np import os import platform import subprocess @@ -67,36 +68,59 @@ def __init__(self, *args, **kwargs): # ---------------------------------------------------------------------------- # JSON EXTENSION # ---------------------------------------------------------------------------- -class ComplexEncoder(json.JSONEncoder): - """A JSON extension for encoding complex numbers +class MatlabEncoder(json.JSONEncoder): + """A JSON extension for encoding numpy arrays to Matlab format - ComplexEncoder encodes complex numbers as a mapping, + Numpy arrays are converted to nested lists. Complex numbers + (either standalone or scalars within an array) are converted to JSON + objects. complex --> {'real': complex.real, 'imag': complex.imag} - """ def default(self, obj): + if isinstance(obj, np.ndarray): + return obj.tolist() if isinstance(obj, complex): return {'real':obj.real, 'imag':obj.imag} # Handle the default case - return json.JSONEncoder.default(self, obj) - -def as_complex(dct): - """A JSON extension for decoding complex numbers - - as_complex decodes mappings of the form {'real': real_val, 'imag': imag_val} - into a single value of type complex + return super(MatlabEncoder, self).default(obj) - Args: - dct: A dictionary - Returns: - complex: A complex number if the dictionary represents an encoding - of a complex number, else the dictionary +class MatlabDecoder(json.JSONDecoder): + """A JSON extension for decoding Matlab arrays into numpy arrays + The default JSON decoder is called first, then elements within the + resulting object tree are greedily marshalled to numpy arrays. If this + fails, the function recurses into """ - if 'real' in dct and 'imag' in dct: - return complex(dct['real'], dct['imag']) - return dct + def __init__(self, encoding='UTF-8', **kwargs): + # register the complex object decoder with the super class + kwargs['object_hook'] = self.decode_complex + # allowable scalar types we can coerce to (not strings or objects) + super(MatlabDecoder, self).__init__(encoding=encoding, **kwargs) + + def decode(self, s): + # decode the string using the default decoder first + tree = super(MatlabDecoder, self).decode(s) + # recursively attempt to build numpy arrays (top-down) + return self.coerce_to_numpy(tree) + + def decode_complex(self, d): + try: + return complex(d['real'], d['imag']) + except KeyError: + return d + + def coerce_to_numpy(self, tree): + """Greedily attempt to coerce an object into a numeric numpy""" + if isinstance(tree, dict): + return dict((key, self.coerce_to_numpy(val)) for key, val in tree.items()) + if isinstance(tree, list): + array = np.array(tree) + if isinstance(array.dtype.type(), (bool, int, float, complex)): + return array + else: + return [self.coerce_to_numpy(item) for item in tree] + return tree # ---------------------------------------------------------------------------- @@ -222,7 +246,7 @@ def stop(self, timeout=1): if not self.started: return - req = json.dumps({'cmd': 'exit'}, cls=ComplexEncoder) + req = json.dumps({'cmd': 'exit'}) try: # the user might be stopping Matlab because the socket is in a bad state self.socket.send(req) @@ -266,7 +290,7 @@ def is_connected(self): if not self.started: raise RuntimeError('No running Matlab subprocess to connect to') - req = json.dumps({'cmd': 'ping'}, cls=ComplexEncoder) + req = json.dumps({'cmd': 'ping'}) self.socket.send(req) start_time = time.time() @@ -327,12 +351,12 @@ def execute_in_matlab(self, req): """ # send the request - req = json.dumps(req, cls=ComplexEncoder) + req = json.dumps(req, cls=MatlabEncoder) self.socket.send(req) # receive the response resp = self.socket.recv_string() - resp = AttributeDict(json.loads(resp, object_hook=as_complex)) + resp = AttributeDict(json.loads(resp, cls=MatlabDecoder)) if not resp.success: raise RuntimeError(resp.result +': '+ resp.message) return resp @@ -382,6 +406,22 @@ def bind_method(self, name, unconditionally=False): return getattr(self, name) + def run_func(self, func_path, func_args=None, maxtime=None): + path, filename = os.path.split(func_path) + func, ext = filename.split('.') + + self.addpath(path) + + def run_code(self, code, maxtime=None): + try: + return {'result': self.eval(code), 'success': 'true', 'message': ''} + except RuntimeError as e: + return {'result': '', 'success': 'false', 'message': e} + + def get_variable(self, varname, maxtime=None): + return self.evalin('base',varname) + + # ---------------------------------------------------------------------------- # MATLAB METHOD # ---------------------------------------------------------------------------- From 69b98cdf09e07150d1bb18df2320cce231492486 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Wed, 9 Apr 2014 16:55:45 +1000 Subject: [PATCH 04/33] Improve socket reboot --- pymatbridge/pymatbridge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index b0b77c7..ee06daa 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -223,6 +223,7 @@ def start(self): self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) # Start the client + self.socket = self.context.socket(zmq.REQ) self.socket.connect(self.socket_addr) self.started = True From dc7cdcad4691daf23854bf89f503d9ed11f72490 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Wed, 9 Apr 2014 23:37:59 +1000 Subject: [PATCH 05/33] Multi-dimensional numpy array support --- pymatbridge/pymatbridge.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index ee06daa..91ba717 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -78,7 +78,9 @@ class MatlabEncoder(json.JSONEncoder): """ def default(self, obj): if isinstance(obj, np.ndarray): - return obj.tolist() + ordering = range(0, obj.ndim) + ordering[0:2] = ordering[1::-1] + return np.transpose(obj, axes=ordering[::-1]).tolist() if isinstance(obj, complex): return {'real':obj.real, 'imag':obj.imag} # Handle the default case @@ -117,7 +119,9 @@ def coerce_to_numpy(self, tree): if isinstance(tree, list): array = np.array(tree) if isinstance(array.dtype.type(), (bool, int, float, complex)): - return array + ordering = range(0, array.ndim) + ordering[-2:] = ordering[:-3:-1] + return np.transpose(array, axes=ordering[::-1]) else: return [self.coerce_to_numpy(item) for item in tree] return tree From e9b94783c719210df2162913cb36322a3dd07b95 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Thu, 10 Apr 2014 11:48:13 +1000 Subject: [PATCH 06/33] Allow graphical plots by default. Draw the plots with a call to matlab.drawnow(). TODO: Allow the EDT to run while zmq is blocking for a request --- pymatbridge/pymatbridge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 91ba717..13142ff 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -173,10 +173,10 @@ def __init__(self, matlab='matlab', socket_addr=None, self.platform = platform if platform else sys.platform if self.platform == 'win32': default_socket_addr = "tcp://127.0.0.1:55555" - default_options = ['-automation', '-nofigureWindows'] + default_options = ['-automation'] else: default_socket_addr = "ipc:///tmp/pymatbridge" - default_options = ['-nodesktop', '-nodisplay'] + default_options = ['-nodesktop', '-nosplash'] self.socket_addr = socket_addr if socket_addr else default_socket_addr self.startup_options = startup_options if startup_options else default_options From df8919b0e92fa00fe2ab3bccf9035c3a01d9e2a3 Mon Sep 17 00:00:00 2001 From: Hilton Bristow Date: Thu, 10 Apr 2014 16:21:43 +1000 Subject: [PATCH 07/33] Migrated setup.py to setuptools. Merged version.py into setup.py. Bumped minor version number to signify new API --- THANKS => CONTRIBUTORS.md | 0 LICENSE | 24 +++++++-- pymatbridge/publish.py | 17 ++++++ scripts/publish-notebook | 17 ------ setup.py | 106 ++++++++++++++++++++++---------------- 5 files changed, 98 insertions(+), 66 deletions(-) rename THANKS => CONTRIBUTORS.md (100%) delete mode 100755 scripts/publish-notebook diff --git a/THANKS b/CONTRIBUTORS.md similarity index 100% rename from THANKS rename to CONTRIBUTORS.md diff --git a/LICENSE b/LICENSE index d3632fc..025ef53 100644 --- a/LICENSE +++ b/LICENSE @@ -1,10 +1,24 @@ -Copyright (c) 2013. See "Contributors". MATLAB (R) is copyright of the Mathworks. +Copyright (c) 2014. See "CONTRIBUTORS.md". +MATLAB (R) is copyright of the Mathworks. All rights reserved. -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: -- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +- Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + -Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pymatbridge/publish.py b/pymatbridge/publish.py index 5843795..d18145e 100644 --- a/pymatbridge/publish.py +++ b/pymatbridge/publish.py @@ -132,3 +132,20 @@ def convert_mfile(mfile, outfile=None): nbformat.write(nb, nbfile, format='ipynb') nbfile.close() + +def main(): + import argparse + + parser = argparse.ArgumentParser( + description='Publish a matlab file (.m) as an interactive notebook (.ipynb)') + + parser.add_argument('mfile', action='store', metavar='File', help='Matlab m-file (.m)') + parser.add_argument('--outfile', action='store', metavar='File', + help='Output notebook (.ipynb). Default: same name and location as the input file ', default=None) + + params = parser.parse_args() + + convert_mfile(params.mfile, params.outfile) + +if __name__ == '__main__': + main() diff --git a/scripts/publish-notebook b/scripts/publish-notebook deleted file mode 100755 index 14e57c5..0000000 --- a/scripts/publish-notebook +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python -import argparse as arg -import pymatbridge.publish as publish - -parser = arg.ArgumentParser(description='Publish a matlab file (.m) as an interactive notebook (.ipynb)') - -parser.add_argument('mfile', action='store', metavar='File', - help='Matlab m-file (.m)') - -parser.add_argument('--outfile', action='store', metavar='File', - help='Output notebook (.ipynb). Default: same name and location as the input file ', default=None) - -params = parser.parse_args() - - -if __name__ == "__main__": - publish.convert_mfile(params.mfile, params.outfile) diff --git a/setup.py b/setup.py index 4c55b21..24ca830 100755 --- a/setup.py +++ b/setup.py @@ -1,25 +1,17 @@ #!/usr/bin/env python -"""Setup file for python-matlab-bridge""" - import os import sys +import glob import shutil +from setuptools import setup -# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly -# update it when the contents of directories change. -if os.path.exists('MANIFEST'): - os.remove('MANIFEST') - -from distutils.core import setup - -# Find the messenger binary file and copy it to /matlab folder. -def copy_bin(bin_path): - if os.path.exists(bin_path): - shutil.copy(bin_path, "./pymatbridge/matlab") - return True - else: - return False +# ---------------------------------------------------------------------------- +# HELPERS +# ---------------------------------------------------------------------------- +def read(file_name): + with open(os.path.join(os.path.dirname(__file__), file_name)) as f: + return f.read() if sys.platform == "win32": raise ValueError("pymatbridge does not work on win32") @@ -27,32 +19,58 @@ def copy_bin(bin_path): for copy_this in ["./messenger/mexmaci64/messenger.mexmaci64", "./messenger/mexa64/messenger.mexa64", "./messenger/mexw64/messenger.mexw64"]: - copy_bin(copy_this) - -# Get version and release info, which is all stored in pymatbridge/version.py -ver_file = os.path.join('pymatbridge', 'version.py') -exec(open(ver_file).read()) - -opts = dict(name=NAME, - maintainer=MAINTAINER, - maintainer_email=MAINTAINER_EMAIL, - description=DESCRIPTION, - long_description=LONG_DESCRIPTION, - url=URL, - download_url=DOWNLOAD_URL, - license=LICENSE, - classifiers=CLASSIFIERS, - author=AUTHOR, - author_email=AUTHOR_EMAIL, - platforms=PLATFORMS, - version=VERSION, - packages=PACKAGES, - package_data=PACKAGE_DATA, - requires=REQUIRES, - scripts=BIN - ) - + shutil.copy(messenger, 'pymatbridge/matlab') + +# ---------------------------------------------------------------------------- +# SETUP +# ---------------------------------------------------------------------------- +__version__ = '0.4.dev' -# Now call the actual setup function -if __name__ == '__main__': - setup(**opts) +setup( + name = 'pymatbridge', + version = __version__, + platforms = 'OS Independent', + description = 'A package to call Matlab functions from Python', + long_description = read('README.md'), + maintainer = 'Ariel Rokem', + maintainer_email = 'arokem@gmail.com', + url = 'https://github.com/arokem/python-matlab-bridge', + download_url = 'https://github.com/arokem/python-matlab-bridge/archive/master.tar.gz', + license = 'BSD', + packages = [ + 'pymatbridge' + ], + install_requires = [ + 'numpy>=1.7.0', + 'pyzmq>=13.0.0' + ], + entry_points = { + 'console_scripts': [ + 'publish-notebook = pymatbridge.publish:main' + ] + }, + package_data = { + 'pymatbridge': [ + 'matlab/matlabserver.m', + 'matlab/messenger.*', + 'matlab/usrprog/*', + 'matlab/util/*.m', + 'matlab/util/json_v0.2.2/LICENSE', + 'matlab/util/json_v0.2.2/README.md', + 'matlab/util/json_v0.2.2/test/*', + 'matlab/util/json_v0.2.2/+json/*.m', + 'matlab/util/json_v0.2.2/+json/java/*', + 'tests/*.py', + 'examples/*.ipynb' + ] + }, + classifiers = [ + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'License :: OSI Approved :: BSD License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Scientific/Engineering' + ] +) From 576ee239953fed97591f09eb0cd149c2b742092a Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:15 +0200 Subject: [PATCH 08/33] Add saveout to function arguments This allows python callers to optionally save variables returned from functions on the MATLAB side. This improves performance as large objects do not need to be serialized so frequently. In combination with changes to run_code and run_script it allows more performant execution of line-by-line MATLAB from python --- pymatbridge/matlab/matlabserver.m | 15 +++++++++++++++ pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 31d1596..ae3af76 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -71,6 +71,7 @@ function matlabserver(socket_address) % TODO: What should the default behaviour be? func = str2func(req.func); nout = req.nout; + [saveout nsaveout] = strsplit(req.saveout,';'); if isempty(nout) try nout = min(abs(nargout(func)), 1); @@ -89,6 +90,20 @@ function matlabserver(socket_address) [resp.result{1:nout}] = func(req.args{:}); end + if length(nsaveout) + if nout == 1 + assignin('base',saveout{1},resp.result); + resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + elseif nout > 1 + tmp_result = ''; + for i=1:nout + assignin('base',saveout{i},resp.result{i}); + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + end + resp.result = tmp_result; + end + end + % build the response resp.success = true; resp.message = 'Successfully completed request'; diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 13142ff..432d51c 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -470,12 +470,22 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): """ # parse out number of output arguments nout = kwargs.pop('nout', None) + saveout = kwargs.pop('saveout',None) + + if nout is None: + saveout = [] + else: + if saveout is not None: + if len(saveout) != nout: + raise ValueError('saveout should be the same length as nout') # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) # build request - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout} + so = ';'.join(saveout) + ';' if saveout else '' + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + resp = self.parent.execute_in_matlab(req) # return the result From 113099b1bdbbe65f330b43642f9c532897d6956f Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:51 +0200 Subject: [PATCH 09/33] Fix run_func to actually call the function --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 432d51c..fff1c66 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -411,11 +411,10 @@ def bind_method(self, name, unconditionally=False): return getattr(self, name) - def run_func(self, func_path, func_args=None, maxtime=None): + def run_func(self, func_path, *args, **kwargs): path, filename = os.path.split(func_path) func, ext = filename.split('.') - - self.addpath(path) + return self.bind_method(func)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From 3a880cf5fb3f3365747684639483a5aeb38f5ffe Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:01:53 +0200 Subject: [PATCH 10/33] Optionally capture stdout Add capture_stdout to constructor. Passing False allows one to see display() etc messages --- pymatbridge/pymatbridge.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index fff1c66..ac3f8f2 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -133,7 +133,8 @@ def coerce_to_numpy(self, tree): class Matlab(object): def __init__(self, matlab='matlab', socket_addr=None, id='python-matlab-bridge', log=False, timeout=30, - platform=None, startup_options=None): + platform=None, startup_options=None, + capture_stdout=True): """Execute functions in a Matlab subprocess via Python Matlab provides a pythonic interface for accessing functions in Matlab. @@ -157,6 +158,8 @@ def __init__(self, matlab='matlab', socket_addr=None, available switches see: Windows: http://www.mathworks.com.au/help/matlab/ref/matlabwindows.html UNIX: http://www.mathworks.com.au/help/matlab/ref/matlabunix.html + capture_stdout: capture (hide) matlab stdout, such as disp() + and redirect to /dev/null/ """ self.MATLAB_FOLDER = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'matlab') @@ -168,6 +171,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.id = id self.log = log self.timeout = timeout + self.capture_stdout = capture_stdout # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -224,7 +228,9 @@ def start(self): command = ' '.join(command) print('Starting Matlab subprocess', end='') - self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) + self.matlab_process = subprocess.Popen(command, shell=True, + stdin=subprocess.PIPE, + stdout=DEVNULL if self.capture_stdout else None) # Start the client self.socket = self.context.socket(zmq.REQ) From cdaded9ccba0d3f6cd456f9b4250fadb94b19852 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:15 +0200 Subject: [PATCH 11/33] Add logging Add log to constructor arguments which shows which functions are called, and what their arguments are --- pymatbridge/pymatbridge.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index ac3f8f2..3afe1e4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -169,9 +169,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.matlab = matlab self.socket_addr = socket_addr self.id = id - self.log = log self.timeout = timeout self.capture_stdout = capture_stdout + self._log = log # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +208,10 @@ def __del__(self): """ self.stop() + def log(self, msg): + if self._log: + print(msg) + def start(self): """Start a new Matlab subprocess and attempt to connect to it via ZMQ @@ -473,6 +477,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): U, S, V = matlab.svd(A, nout=3) """ + self.parent.log("CALL: %s" % self.name) + # parse out number of output arguments nout = kwargs.pop('nout', None) saveout = kwargs.pop('saveout',None) @@ -491,6 +497,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): so = ';'.join(saveout) + ';' if saveout else '' req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + self.parent.log("REQ: %r:"%req) + resp = self.parent.execute_in_matlab(req) # return the result From 9f4265f74447f5af3f54bfc20b33690ad615a6ad Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:23 +0200 Subject: [PATCH 12/33] Add a path cache Because run_code, run_script, and run_function all manipulate the MATLAB path add a local python path cache to reduce the number of addpath calls --- pymatbridge/pymatbridge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 3afe1e4..5aab904 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -172,6 +172,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.timeout = timeout self.capture_stdout = capture_stdout self._log = log + self._path_cache = None # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +209,21 @@ def __del__(self): """ self.stop() + def _ensure_in_path(self, path): + if not os.path.isfile(path): + raise ValueError("not a valid matlab file: %s" % path) + + path, filename = os.path.split(path) + funcname, ext = os.path.splitext(filename) + + if self._path_cache is None: + self._path_cache = self.path().split(os.pathsep) + if path not in self._path_cache: + self.addpath(path) + self._path_cache.append(path) + + return path,funcname + def log(self, msg): if self._log: print(msg) From c8db975a718ec4d924196b4bfb5524a7fead6df6 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:05 +0200 Subject: [PATCH 13/33] Remove weakref * It didnt actually work with functions with multiple arguments * Log function proxy creation --- pymatbridge/pymatbridge.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 5aab904..6cbd058 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -433,7 +433,7 @@ def bind_method(self, name, unconditionally=False): method_instance.__name__ = name # bind to the Matlab instance with a weakref (to avoid circular references) - setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + setattr(self, name, types.MethodType(method_instance, Matlab)) return getattr(self, name) @@ -469,11 +469,13 @@ def __init__(self, parent, name): name: The name of the Matlab function this represents """ - self.name = name self._parent = parent + self.name = name self.doc = None - def __call__(self, unused_parent_weakref, *args, **kwargs): + self.parent.log("CREATED: %s" % self.name) + + def __call__(self, _, *args, **kwargs): """Call a function with the supplied arguments in the Matlab subprocess Args: From a07604a759d905b106f008bf97b80a59e00364f3 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:30 +0200 Subject: [PATCH 14/33] Fix run_func to add the path --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 6cbd058..db068fb 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -438,9 +438,8 @@ def bind_method(self, name, unconditionally=False): def run_func(self, func_path, *args, **kwargs): - path, filename = os.path.split(func_path) - func, ext = filename.split('.') - return self.bind_method(func)(*args, **kwargs) + path, funcname = self._ensure_in_path(func_path) + return self.bind_method(funcname)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From acaf1b7bfe071670e6e762e37c3e95e8d57e2104 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:52 +0200 Subject: [PATCH 15/33] Run_code in the base workspace * So we can get to the variables created within --- pymatbridge/pymatbridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index db068fb..b3a7fc4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -443,7 +443,7 @@ def run_func(self, func_path, *args, **kwargs): def run_code(self, code, maxtime=None): try: - return {'result': self.eval(code), 'success': 'true', 'message': ''} + return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} except RuntimeError as e: return {'result': '', 'success': 'false', 'message': e} From 7d38469b00f3eb32d1251e9586bf73e7cd85b589 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 18:56:00 +0200 Subject: [PATCH 16/33] Pass and return args as dictionaries Due to the inherent ambiguity in matlab's JSON decoding ability this is the only way to make passing >1 same-sized array arguments work. Otherwise the matlab json library coerces them to a matrix --- pymatbridge/matlab/matlabserver.m | 29 +++++++++++++++++++++-------- pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index ae3af76..7a44af0 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -62,9 +62,11 @@ function matlabserver(socket_address) % 'args': A cell array of args to expand into the function arguments % 'nout': The number of output arguments requested - % function of no arguments - if ~isfield(req, 'args') - req.args = {} + args = {}; + if req.nin > 0 + for i=1:req.nin + args{i} = req.args.(['a' num2str(i-1)]); + end end % determine the number of output arguments @@ -83,22 +85,33 @@ function matlabserver(socket_address) % call the function, taking care of broadcasting outputs switch nout case 0 - func(req.args{:}); + func(args{:}); case 1 - resp.result = func(req.args{:}); + resp.result = func(args{:}); otherwise - [resp.result{1:nout}] = func(req.args{:}); + [resp.result{1:nout}] = func(args{:}); + if ~length(nsaveout) + %because of ambiguity of json encoding arrays of matrices + %convert multiple output arguments into a structure with + %fields name a0..aN. Convert these back to a list of matrices + %at the python end + result_struct = struct('nout',nout); + for i=1:nout + result_struct.(['a' num2str(i-1)]) = resp.result{i}; + end + resp.result = result_struct; + end end if length(nsaveout) if nout == 1 assignin('base',saveout{1},resp.result); - resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + resp.result = ['__VAR=' saveout{1} '|' class(resp.result) '(' mat2str(size(resp.result)) ')']; elseif nout > 1 tmp_result = ''; for i=1:nout assignin('base',saveout{i},resp.result{i}); - tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) '(' mat2str(size(resp.result{i})) ');' tmp_result]; end resp.result = tmp_result; end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index b3a7fc4..79c6008 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -390,6 +390,10 @@ def execute_in_matlab(self, req): resp = AttributeDict(json.loads(resp, cls=MatlabDecoder)) if not resp.success: raise RuntimeError(resp.result +': '+ resp.message) + + if hasattr(resp, 'result') and isinstance(resp.result, dict) and 'nout' in resp.result: + resp.result = [resp.result['a%d'%i] for i in range(resp.result['nout'])] + return resp def __getattr__(self, name): @@ -510,9 +514,15 @@ def __call__(self, _, *args, **kwargs): # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) + #now convert to a dict with string(num) keys because of the ambiguity + #of JSON wrt decoding [[1,2],[3,4]] (2 array args get decoded as a single + #matrix argument + nin = len(args) + args = {'a%d'%i:a for i,a in enumerate(args)} + # build request so = ';'.join(saveout) + ';' if saveout else '' - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nin': nin, 'nout': nout, 'saveout': so} self.parent.log("REQ: %r:"%req) From 39936790679e2f3a5e492ee37702f220124dd660 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:05:49 +0200 Subject: [PATCH 17/33] Add run_script Runs a m script file and stores any variables in the base workspace --- pymatbridge/pymatbridge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 79c6008..bfc91da 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -445,6 +445,10 @@ def run_func(self, func_path, *args, **kwargs): path, funcname = self._ensure_in_path(func_path) return self.bind_method(funcname)(*args, **kwargs) + def run_script(self, script_path): + path, funcname = self._ensure_in_path(script_path) + self.evalin('base',"run('%s')" % funcname, nout=0) + def run_code(self, code, maxtime=None): try: return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} From e315ad0862bdcb4c2bf409ede2d1e3b269fcf50e Mon Sep 17 00:00:00 2001 From: John Stowers Date: Mon, 4 Aug 2014 16:58:19 +0200 Subject: [PATCH 18/33] Rework run_code Write a temporary m file and execute it via run_script to allows variables to persist in the base workspace. --- pymatbridge/pymatbridge.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index bfc91da..f5b18b8 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -18,6 +18,9 @@ import types import weakref import zmq +import tempfile +import hashlib +import shutil try: # Python 2 basestring @@ -199,6 +202,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.bind_method('run', unconditionally=True) self.bind_method('version', unconditionally=True) + #generate a temporary directory to run code in + self.tempdir_code = tempfile.mkdtemp(prefix='pymatlabridge',suffix='code') + def __del__(self): """Forcibly cleanup resources @@ -296,6 +302,8 @@ def stop(self, timeout=1): self.socket.close() self.started = False + shutil.rmtree(self.tempdir_code) + def restart(self): """Restart the Matlab subprocess if the state becomes bad @@ -447,13 +455,16 @@ def run_func(self, func_path, *args, **kwargs): def run_script(self, script_path): path, funcname = self._ensure_in_path(script_path) - self.evalin('base',"run('%s')" % funcname, nout=0) - - def run_code(self, code, maxtime=None): - try: - return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} - except RuntimeError as e: - return {'result': '', 'success': 'false', 'message': e} + return self.evalin('base',"run('%s')" % funcname, nout=0) + + def run_code(self, code): + #write a temporary file + fn = os.path.join(self.tempdir_code, + 'code_' + hashlib.md5(code).hexdigest() + '.m') + if not os.path.isfile(fn): + with open(fn,'w') as f: + f.write(code) + return self.run_script(fn) def get_variable(self, varname, maxtime=None): return self.evalin('base',varname) From 5652b4ea78d444cc6304ae1321fe399423c53f67 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 5 Aug 2014 11:49:47 +0200 Subject: [PATCH 19/33] Revert "Migrated setup.py to setuptools. This reverts commit df8919b0e92fa00fe2ab3bccf9035c3a01d9e2a3. It didn't actually work. --- LICENSE | 24 ++------- CONTRIBUTORS.md => THANKS | 0 pymatbridge/publish.py | 17 ------ scripts/publish-notebook | 17 ++++++ setup.py | 106 ++++++++++++++++---------------------- 5 files changed, 66 insertions(+), 98 deletions(-) rename CONTRIBUTORS.md => THANKS (100%) create mode 100755 scripts/publish-notebook diff --git a/LICENSE b/LICENSE index 025ef53..d3632fc 100644 --- a/LICENSE +++ b/LICENSE @@ -1,24 +1,10 @@ -Copyright (c) 2014. See "CONTRIBUTORS.md". -MATLAB (R) is copyright of the Mathworks. +Copyright (c) 2013. See "Contributors". MATLAB (R) is copyright of the Mathworks. All rights reserved. -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -- Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. +- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + -Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/CONTRIBUTORS.md b/THANKS similarity index 100% rename from CONTRIBUTORS.md rename to THANKS diff --git a/pymatbridge/publish.py b/pymatbridge/publish.py index d18145e..5843795 100644 --- a/pymatbridge/publish.py +++ b/pymatbridge/publish.py @@ -132,20 +132,3 @@ def convert_mfile(mfile, outfile=None): nbformat.write(nb, nbfile, format='ipynb') nbfile.close() - -def main(): - import argparse - - parser = argparse.ArgumentParser( - description='Publish a matlab file (.m) as an interactive notebook (.ipynb)') - - parser.add_argument('mfile', action='store', metavar='File', help='Matlab m-file (.m)') - parser.add_argument('--outfile', action='store', metavar='File', - help='Output notebook (.ipynb). Default: same name and location as the input file ', default=None) - - params = parser.parse_args() - - convert_mfile(params.mfile, params.outfile) - -if __name__ == '__main__': - main() diff --git a/scripts/publish-notebook b/scripts/publish-notebook new file mode 100755 index 0000000..14e57c5 --- /dev/null +++ b/scripts/publish-notebook @@ -0,0 +1,17 @@ +#!/usr/bin/env python +import argparse as arg +import pymatbridge.publish as publish + +parser = arg.ArgumentParser(description='Publish a matlab file (.m) as an interactive notebook (.ipynb)') + +parser.add_argument('mfile', action='store', metavar='File', + help='Matlab m-file (.m)') + +parser.add_argument('--outfile', action='store', metavar='File', + help='Output notebook (.ipynb). Default: same name and location as the input file ', default=None) + +params = parser.parse_args() + + +if __name__ == "__main__": + publish.convert_mfile(params.mfile, params.outfile) diff --git a/setup.py b/setup.py index 24ca830..4c55b21 100755 --- a/setup.py +++ b/setup.py @@ -1,17 +1,25 @@ #!/usr/bin/env python +"""Setup file for python-matlab-bridge""" + import os import sys -import glob import shutil -from setuptools import setup +# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly +# update it when the contents of directories change. +if os.path.exists('MANIFEST'): + os.remove('MANIFEST') + +from distutils.core import setup + +# Find the messenger binary file and copy it to /matlab folder. -# ---------------------------------------------------------------------------- -# HELPERS -# ---------------------------------------------------------------------------- -def read(file_name): - with open(os.path.join(os.path.dirname(__file__), file_name)) as f: - return f.read() +def copy_bin(bin_path): + if os.path.exists(bin_path): + shutil.copy(bin_path, "./pymatbridge/matlab") + return True + else: + return False if sys.platform == "win32": raise ValueError("pymatbridge does not work on win32") @@ -19,58 +27,32 @@ def read(file_name): for copy_this in ["./messenger/mexmaci64/messenger.mexmaci64", "./messenger/mexa64/messenger.mexa64", "./messenger/mexw64/messenger.mexw64"]: - shutil.copy(messenger, 'pymatbridge/matlab') - -# ---------------------------------------------------------------------------- -# SETUP -# ---------------------------------------------------------------------------- -__version__ = '0.4.dev' + copy_bin(copy_this) + +# Get version and release info, which is all stored in pymatbridge/version.py +ver_file = os.path.join('pymatbridge', 'version.py') +exec(open(ver_file).read()) + +opts = dict(name=NAME, + maintainer=MAINTAINER, + maintainer_email=MAINTAINER_EMAIL, + description=DESCRIPTION, + long_description=LONG_DESCRIPTION, + url=URL, + download_url=DOWNLOAD_URL, + license=LICENSE, + classifiers=CLASSIFIERS, + author=AUTHOR, + author_email=AUTHOR_EMAIL, + platforms=PLATFORMS, + version=VERSION, + packages=PACKAGES, + package_data=PACKAGE_DATA, + requires=REQUIRES, + scripts=BIN + ) + -setup( - name = 'pymatbridge', - version = __version__, - platforms = 'OS Independent', - description = 'A package to call Matlab functions from Python', - long_description = read('README.md'), - maintainer = 'Ariel Rokem', - maintainer_email = 'arokem@gmail.com', - url = 'https://github.com/arokem/python-matlab-bridge', - download_url = 'https://github.com/arokem/python-matlab-bridge/archive/master.tar.gz', - license = 'BSD', - packages = [ - 'pymatbridge' - ], - install_requires = [ - 'numpy>=1.7.0', - 'pyzmq>=13.0.0' - ], - entry_points = { - 'console_scripts': [ - 'publish-notebook = pymatbridge.publish:main' - ] - }, - package_data = { - 'pymatbridge': [ - 'matlab/matlabserver.m', - 'matlab/messenger.*', - 'matlab/usrprog/*', - 'matlab/util/*.m', - 'matlab/util/json_v0.2.2/LICENSE', - 'matlab/util/json_v0.2.2/README.md', - 'matlab/util/json_v0.2.2/test/*', - 'matlab/util/json_v0.2.2/+json/*.m', - 'matlab/util/json_v0.2.2/+json/java/*', - 'tests/*.py', - 'examples/*.ipynb' - ] - }, - classifiers = [ - 'Development Status :: 3 - Alpha', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'License :: OSI Approved :: BSD License', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Topic :: Scientific/Engineering' - ] -) +# Now call the actual setup function +if __name__ == '__main__': + setup(**opts) From 06f3888fa017f09875238ced3ed6625d73cdebbe Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 5 Aug 2014 11:48:46 +0200 Subject: [PATCH 20/33] Add examples of recent functionality --- pymatbridge/examples/example_func.m | 3 ++ pymatbridge/examples/example_script.m | 3 ++ pymatbridge/examples/outvariables.py | 57 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 pymatbridge/examples/example_func.m create mode 100644 pymatbridge/examples/example_script.m create mode 100644 pymatbridge/examples/outvariables.py diff --git a/pymatbridge/examples/example_func.m b/pymatbridge/examples/example_func.m new file mode 100644 index 0000000..063fbaf --- /dev/null +++ b/pymatbridge/examples/example_func.m @@ -0,0 +1,3 @@ +function lol = bar_func(name) + lol=['hello from ' name]; +end diff --git a/pymatbridge/examples/example_script.m b/pymatbridge/examples/example_script.m new file mode 100644 index 0000000..ae468e9 --- /dev/null +++ b/pymatbridge/examples/example_script.m @@ -0,0 +1,3 @@ +q=1:10; +f=45; + diff --git a/pymatbridge/examples/outvariables.py b/pymatbridge/examples/outvariables.py new file mode 100644 index 0000000..4502753 --- /dev/null +++ b/pymatbridge/examples/outvariables.py @@ -0,0 +1,57 @@ +import time +import numpy as np +import numpy.testing as npt +import os.path +import pymatbridge + +_dir = os.path.dirname(os.path.abspath(__file__)) + +mlab = pymatbridge.Matlab(matlab='/opt/matlab/R2013a/bin/matlab', log=True, capture_stdout=True) +mlab.start() + +if 1: + #conventional behaviour, perform the matlab command and return the result + z = mlab.zeros(5) + npt.assert_equal(z, np.zeros((5,5))) + + #perform the same command, and save the 1 output variable on the matlab side + #with the name 'z'. return a placeholder containing some metadata about it + placeholder = mlab.zeros(5,nout=1,saveout=('z',)) + assert placeholder == '__VAR=z|double([5 5])' + #now return the result + z = mlab.get_variable('z') + npt.assert_equal(z, np.zeros((5,5))) + +if 1: + #this time the matlab command returns two variables + x,y = mlab.meshgrid(range(1,4),range(10,15),nout=2) + npx,npy = np.meshgrid(range(1,4),range(10,15)) + npt.assert_equal(x,npx); npt.assert_equal(y,npy) + + #perform the same command, but leave the result in matlab + placeholder = mlab.meshgrid(range(1,4),range(10,15),nout=2,saveout=('X','Y')) + assert placeholder == '__VAR=Y|double([5 3]);__VAR=X|double([5 3]);' + + #now return the result + x = mlab.get_variable('X') + npt.assert_equal(x,npx) + +if 1: + mlab.run_func(os.path.join(_dir,'example_func.m'), 'john', nout=1, saveout=('lol',)) + assert 'hello from john' == mlab.get_variable('lol') + +if 1: + mlab.run_script(os.path.join(_dir,'example_script.m')) + q = mlab.get_variable('q') + npt.assert_equal(q,range(1,11)) + f = mlab.get_variable('f') + npt.assert_equal(f,45) + +if 1: + mlab.run_code('foo=1:100;') + m = mlab.get_variable('foo') + npt.assert_equal(m,range(1,101)) + + mlab.run_code('foo=1:100;') + +mlab.stop() From 65d2cec36f273e3cd3f0404500bb3be4a082a15d Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:03:30 +0200 Subject: [PATCH 21/33] Send a better exception report to the python side Shows a full MATLAB traceback in case of error --- pymatbridge/matlab/matlabserver.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 279f54d..31d1596 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -41,7 +41,7 @@ function matlabserver(socket_address) % format the exception and pass it back to the client resp.success = false; resp.result = exception.identifier; - resp.message = exception.message; + resp.message = getReport(exception) json_resp = json.dump(resp); messenger('respond', json_resp); From fcf58c99aeab194ced5a2ef5e9cca9c0f66aed48 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:15 +0200 Subject: [PATCH 22/33] Add saveout to function arguments This allows python callers to optionally save variables returned from functions on the MATLAB side. This improves performance as large objects do not need to be serialized so frequently. In combination with changes to run_code and run_script it allows more performant execution of line-by-line MATLAB from python --- pymatbridge/matlab/matlabserver.m | 15 +++++++++++++++ pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index 31d1596..a7b0faa 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -71,6 +71,7 @@ function matlabserver(socket_address) % TODO: What should the default behaviour be? func = str2func(req.func); nout = req.nout; + [saveout nsaveout] = regexp(req.saveout, '(?:;)+', 'split', 'match'); if isempty(nout) try nout = min(abs(nargout(func)), 1); @@ -89,6 +90,20 @@ function matlabserver(socket_address) [resp.result{1:nout}] = func(req.args{:}); end + if length(nsaveout) + if nout == 1 + assignin('base',saveout{1},resp.result); + resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + elseif nout > 1 + tmp_result = ''; + for i=1:nout + assignin('base',saveout{i},resp.result{i}); + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + end + resp.result = tmp_result; + end + end + % build the response resp.success = true; resp.message = 'Successfully completed request'; diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 13142ff..432d51c 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -470,12 +470,22 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): """ # parse out number of output arguments nout = kwargs.pop('nout', None) + saveout = kwargs.pop('saveout',None) + + if nout is None: + saveout = [] + else: + if saveout is not None: + if len(saveout) != nout: + raise ValueError('saveout should be the same length as nout') # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) # build request - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout} + so = ';'.join(saveout) + ';' if saveout else '' + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + resp = self.parent.execute_in_matlab(req) # return the result From f8a0ef03ed0b9b08765a9f2d3d8585515551098e Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 19:04:51 +0200 Subject: [PATCH 23/33] Fix run_func to actually call the function --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 432d51c..fff1c66 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -411,11 +411,10 @@ def bind_method(self, name, unconditionally=False): return getattr(self, name) - def run_func(self, func_path, func_args=None, maxtime=None): + def run_func(self, func_path, *args, **kwargs): path, filename = os.path.split(func_path) func, ext = filename.split('.') - - self.addpath(path) + return self.bind_method(func)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From cc6376b1e2ff0c5794092ffabf58f5f493d2b5d8 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:01:53 +0200 Subject: [PATCH 24/33] Optionally capture stdout Add capture_stdout to constructor. Passing False allows one to see display() etc messages --- pymatbridge/pymatbridge.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index fff1c66..ac3f8f2 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -133,7 +133,8 @@ def coerce_to_numpy(self, tree): class Matlab(object): def __init__(self, matlab='matlab', socket_addr=None, id='python-matlab-bridge', log=False, timeout=30, - platform=None, startup_options=None): + platform=None, startup_options=None, + capture_stdout=True): """Execute functions in a Matlab subprocess via Python Matlab provides a pythonic interface for accessing functions in Matlab. @@ -157,6 +158,8 @@ def __init__(self, matlab='matlab', socket_addr=None, available switches see: Windows: http://www.mathworks.com.au/help/matlab/ref/matlabwindows.html UNIX: http://www.mathworks.com.au/help/matlab/ref/matlabunix.html + capture_stdout: capture (hide) matlab stdout, such as disp() + and redirect to /dev/null/ """ self.MATLAB_FOLDER = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'matlab') @@ -168,6 +171,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.id = id self.log = log self.timeout = timeout + self.capture_stdout = capture_stdout # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -224,7 +228,9 @@ def start(self): command = ' '.join(command) print('Starting Matlab subprocess', end='') - self.matlab_process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, stdout=DEVNULL) + self.matlab_process = subprocess.Popen(command, shell=True, + stdin=subprocess.PIPE, + stdout=DEVNULL if self.capture_stdout else None) # Start the client self.socket = self.context.socket(zmq.REQ) From e9165829ca857bc9e73ade53fe8679d323fda009 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:15 +0200 Subject: [PATCH 25/33] Add logging Add log to constructor arguments which shows which functions are called, and what their arguments are --- pymatbridge/pymatbridge.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index ac3f8f2..3afe1e4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -169,9 +169,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.matlab = matlab self.socket_addr = socket_addr self.id = id - self.log = log self.timeout = timeout self.capture_stdout = capture_stdout + self._log = log # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +208,10 @@ def __del__(self): """ self.stop() + def log(self, msg): + if self._log: + print(msg) + def start(self): """Start a new Matlab subprocess and attempt to connect to it via ZMQ @@ -473,6 +477,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): U, S, V = matlab.svd(A, nout=3) """ + self.parent.log("CALL: %s" % self.name) + # parse out number of output arguments nout = kwargs.pop('nout', None) saveout = kwargs.pop('saveout',None) @@ -491,6 +497,8 @@ def __call__(self, unused_parent_weakref, *args, **kwargs): so = ';'.join(saveout) + ';' if saveout else '' req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + self.parent.log("REQ: %r:"%req) + resp = self.parent.execute_in_matlab(req) # return the result From 6048a916a0a8702db16f8c41d92437ccd74fdfa0 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:02:23 +0200 Subject: [PATCH 26/33] Add a path cache Because run_code, run_script, and run_function all manipulate the MATLAB path add a local python path cache to reduce the number of addpath calls --- pymatbridge/pymatbridge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 3afe1e4..5aab904 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -172,6 +172,7 @@ def __init__(self, matlab='matlab', socket_addr=None, self.timeout = timeout self.capture_stdout = capture_stdout self._log = log + self._path_cache = None # determine the platform-specific options self.platform = platform if platform else sys.platform @@ -208,6 +209,21 @@ def __del__(self): """ self.stop() + def _ensure_in_path(self, path): + if not os.path.isfile(path): + raise ValueError("not a valid matlab file: %s" % path) + + path, filename = os.path.split(path) + funcname, ext = os.path.splitext(filename) + + if self._path_cache is None: + self._path_cache = self.path().split(os.pathsep) + if path not in self._path_cache: + self.addpath(path) + self._path_cache.append(path) + + return path,funcname + def log(self, msg): if self._log: print(msg) From 1102515834e7a620d7e5525ce7c678d43d1c5542 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:05 +0200 Subject: [PATCH 27/33] Remove weakref * It didnt actually work with functions with multiple arguments * Log function proxy creation --- pymatbridge/pymatbridge.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 5aab904..6cbd058 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -433,7 +433,7 @@ def bind_method(self, name, unconditionally=False): method_instance.__name__ = name # bind to the Matlab instance with a weakref (to avoid circular references) - setattr(self, name, types.MethodType(method_instance, weakref.ref(self), Matlab)) + setattr(self, name, types.MethodType(method_instance, Matlab)) return getattr(self, name) @@ -469,11 +469,13 @@ def __init__(self, parent, name): name: The name of the Matlab function this represents """ - self.name = name self._parent = parent + self.name = name self.doc = None - def __call__(self, unused_parent_weakref, *args, **kwargs): + self.parent.log("CREATED: %s" % self.name) + + def __call__(self, _, *args, **kwargs): """Call a function with the supplied arguments in the Matlab subprocess Args: From e5ddc189b8fc5dfa9ba9adc5c10c4cc6f644f562 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:30 +0200 Subject: [PATCH 28/33] Fix run_func to add the path --- pymatbridge/pymatbridge.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 6cbd058..db068fb 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -438,9 +438,8 @@ def bind_method(self, name, unconditionally=False): def run_func(self, func_path, *args, **kwargs): - path, filename = os.path.split(func_path) - func, ext = filename.split('.') - return self.bind_method(func)(*args, **kwargs) + path, funcname = self._ensure_in_path(func_path) + return self.bind_method(funcname)(*args, **kwargs) def run_code(self, code, maxtime=None): try: From 2e7b082f73ada9555b3c1d857496d9f58e804bc1 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:07:52 +0200 Subject: [PATCH 29/33] Run_code in the base workspace * So we can get to the variables created within --- pymatbridge/pymatbridge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index db068fb..b3a7fc4 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -443,7 +443,7 @@ def run_func(self, func_path, *args, **kwargs): def run_code(self, code, maxtime=None): try: - return {'result': self.eval(code), 'success': 'true', 'message': ''} + return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} except RuntimeError as e: return {'result': '', 'success': 'false', 'message': e} From 4871740bedc4ab4e642588ce8e350870ac5a938f Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 8 Jul 2014 18:56:00 +0200 Subject: [PATCH 30/33] Pass and return args as dictionaries Due to the inherent ambiguity in matlab's JSON decoding ability this is the only way to make passing >1 same-sized array arguments work. Otherwise the matlab json library coerces them to a matrix --- pymatbridge/matlab/matlabserver.m | 29 +++++++++++++++++++++-------- pymatbridge/pymatbridge.py | 12 +++++++++++- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/pymatbridge/matlab/matlabserver.m b/pymatbridge/matlab/matlabserver.m index a7b0faa..066e185 100644 --- a/pymatbridge/matlab/matlabserver.m +++ b/pymatbridge/matlab/matlabserver.m @@ -62,9 +62,11 @@ function matlabserver(socket_address) % 'args': A cell array of args to expand into the function arguments % 'nout': The number of output arguments requested - % function of no arguments - if ~isfield(req, 'args') - req.args = {} + args = {}; + if req.nin > 0 + for i=1:req.nin + args{i} = req.args.(['a' num2str(i-1)]); + end end % determine the number of output arguments @@ -83,22 +85,33 @@ function matlabserver(socket_address) % call the function, taking care of broadcasting outputs switch nout case 0 - func(req.args{:}); + func(args{:}); case 1 - resp.result = func(req.args{:}); + resp.result = func(args{:}); otherwise - [resp.result{1:nout}] = func(req.args{:}); + [resp.result{1:nout}] = func(args{:}); + if ~length(nsaveout) + %because of ambiguity of json encoding arrays of matrices + %convert multiple output arguments into a structure with + %fields name a0..aN. Convert these back to a list of matrices + %at the python end + result_struct = struct('nout',nout); + for i=1:nout + result_struct.(['a' num2str(i-1)]) = resp.result{i}; + end + resp.result = result_struct; + end end if length(nsaveout) if nout == 1 assignin('base',saveout{1},resp.result); - resp.result = ['__VAR=' saveout{1} '|' class(resp.result)]; + resp.result = ['__VAR=' saveout{1} '|' class(resp.result) '(' mat2str(size(resp.result)) ')']; elseif nout > 1 tmp_result = ''; for i=1:nout assignin('base',saveout{i},resp.result{i}); - tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) ';' tmp_result]; + tmp_result = ['__VAR=' saveout{i} '|' class(resp.result{i}) '(' mat2str(size(resp.result{i})) ');' tmp_result]; end resp.result = tmp_result; end diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index b3a7fc4..79c6008 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -390,6 +390,10 @@ def execute_in_matlab(self, req): resp = AttributeDict(json.loads(resp, cls=MatlabDecoder)) if not resp.success: raise RuntimeError(resp.result +': '+ resp.message) + + if hasattr(resp, 'result') and isinstance(resp.result, dict) and 'nout' in resp.result: + resp.result = [resp.result['a%d'%i] for i in range(resp.result['nout'])] + return resp def __getattr__(self, name): @@ -510,9 +514,15 @@ def __call__(self, _, *args, **kwargs): # convert keyword arguments to arguments args += tuple(item for pair in zip(kwargs.keys(), kwargs.values()) for item in pair) + #now convert to a dict with string(num) keys because of the ambiguity + #of JSON wrt decoding [[1,2],[3,4]] (2 array args get decoded as a single + #matrix argument + nin = len(args) + args = {'a%d'%i:a for i,a in enumerate(args)} + # build request so = ';'.join(saveout) + ';' if saveout else '' - req = {'cmd': 'call', 'func': self.name, 'args': args, 'nout': nout, 'saveout': so} + req = {'cmd': 'call', 'func': self.name, 'args': args, 'nin': nin, 'nout': nout, 'saveout': so} self.parent.log("REQ: %r:"%req) From be28bd949e86ba057f2f9def2aab197a47649737 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Thu, 10 Jul 2014 15:05:49 +0200 Subject: [PATCH 31/33] Add run_script Runs a m script file and stores any variables in the base workspace --- pymatbridge/pymatbridge.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index 79c6008..bfc91da 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -445,6 +445,10 @@ def run_func(self, func_path, *args, **kwargs): path, funcname = self._ensure_in_path(func_path) return self.bind_method(funcname)(*args, **kwargs) + def run_script(self, script_path): + path, funcname = self._ensure_in_path(script_path) + self.evalin('base',"run('%s')" % funcname, nout=0) + def run_code(self, code, maxtime=None): try: return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} From d53120b992ff215adf51160987165851c852f0c6 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Mon, 4 Aug 2014 16:58:19 +0200 Subject: [PATCH 32/33] Rework run_code Write a temporary m file and execute it via run_script to allows variables to persist in the base workspace. --- pymatbridge/pymatbridge.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/pymatbridge/pymatbridge.py b/pymatbridge/pymatbridge.py index bfc91da..f5b18b8 100644 --- a/pymatbridge/pymatbridge.py +++ b/pymatbridge/pymatbridge.py @@ -18,6 +18,9 @@ import types import weakref import zmq +import tempfile +import hashlib +import shutil try: # Python 2 basestring @@ -199,6 +202,9 @@ def __init__(self, matlab='matlab', socket_addr=None, self.bind_method('run', unconditionally=True) self.bind_method('version', unconditionally=True) + #generate a temporary directory to run code in + self.tempdir_code = tempfile.mkdtemp(prefix='pymatlabridge',suffix='code') + def __del__(self): """Forcibly cleanup resources @@ -296,6 +302,8 @@ def stop(self, timeout=1): self.socket.close() self.started = False + shutil.rmtree(self.tempdir_code) + def restart(self): """Restart the Matlab subprocess if the state becomes bad @@ -447,13 +455,16 @@ def run_func(self, func_path, *args, **kwargs): def run_script(self, script_path): path, funcname = self._ensure_in_path(script_path) - self.evalin('base',"run('%s')" % funcname, nout=0) - - def run_code(self, code, maxtime=None): - try: - return {'result': self.evalin('base',code), 'success': 'true', 'message': ''} - except RuntimeError as e: - return {'result': '', 'success': 'false', 'message': e} + return self.evalin('base',"run('%s')" % funcname, nout=0) + + def run_code(self, code): + #write a temporary file + fn = os.path.join(self.tempdir_code, + 'code_' + hashlib.md5(code).hexdigest() + '.m') + if not os.path.isfile(fn): + with open(fn,'w') as f: + f.write(code) + return self.run_script(fn) def get_variable(self, varname, maxtime=None): return self.evalin('base',varname) From 2004b7e502d9efcc1c48ecdf32513c3723dd7020 Mon Sep 17 00:00:00 2001 From: John Stowers Date: Tue, 5 Aug 2014 11:48:46 +0200 Subject: [PATCH 33/33] Add examples of recent functionality --- pymatbridge/examples/example_func.m | 3 ++ pymatbridge/examples/example_script.m | 3 ++ pymatbridge/examples/outvariables.py | 57 +++++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 pymatbridge/examples/example_func.m create mode 100644 pymatbridge/examples/example_script.m create mode 100644 pymatbridge/examples/outvariables.py diff --git a/pymatbridge/examples/example_func.m b/pymatbridge/examples/example_func.m new file mode 100644 index 0000000..063fbaf --- /dev/null +++ b/pymatbridge/examples/example_func.m @@ -0,0 +1,3 @@ +function lol = bar_func(name) + lol=['hello from ' name]; +end diff --git a/pymatbridge/examples/example_script.m b/pymatbridge/examples/example_script.m new file mode 100644 index 0000000..ae468e9 --- /dev/null +++ b/pymatbridge/examples/example_script.m @@ -0,0 +1,3 @@ +q=1:10; +f=45; + diff --git a/pymatbridge/examples/outvariables.py b/pymatbridge/examples/outvariables.py new file mode 100644 index 0000000..4502753 --- /dev/null +++ b/pymatbridge/examples/outvariables.py @@ -0,0 +1,57 @@ +import time +import numpy as np +import numpy.testing as npt +import os.path +import pymatbridge + +_dir = os.path.dirname(os.path.abspath(__file__)) + +mlab = pymatbridge.Matlab(matlab='/opt/matlab/R2013a/bin/matlab', log=True, capture_stdout=True) +mlab.start() + +if 1: + #conventional behaviour, perform the matlab command and return the result + z = mlab.zeros(5) + npt.assert_equal(z, np.zeros((5,5))) + + #perform the same command, and save the 1 output variable on the matlab side + #with the name 'z'. return a placeholder containing some metadata about it + placeholder = mlab.zeros(5,nout=1,saveout=('z',)) + assert placeholder == '__VAR=z|double([5 5])' + #now return the result + z = mlab.get_variable('z') + npt.assert_equal(z, np.zeros((5,5))) + +if 1: + #this time the matlab command returns two variables + x,y = mlab.meshgrid(range(1,4),range(10,15),nout=2) + npx,npy = np.meshgrid(range(1,4),range(10,15)) + npt.assert_equal(x,npx); npt.assert_equal(y,npy) + + #perform the same command, but leave the result in matlab + placeholder = mlab.meshgrid(range(1,4),range(10,15),nout=2,saveout=('X','Y')) + assert placeholder == '__VAR=Y|double([5 3]);__VAR=X|double([5 3]);' + + #now return the result + x = mlab.get_variable('X') + npt.assert_equal(x,npx) + +if 1: + mlab.run_func(os.path.join(_dir,'example_func.m'), 'john', nout=1, saveout=('lol',)) + assert 'hello from john' == mlab.get_variable('lol') + +if 1: + mlab.run_script(os.path.join(_dir,'example_script.m')) + q = mlab.get_variable('q') + npt.assert_equal(q,range(1,11)) + f = mlab.get_variable('f') + npt.assert_equal(f,45) + +if 1: + mlab.run_code('foo=1:100;') + m = mlab.get_variable('foo') + npt.assert_equal(m,range(1,101)) + + mlab.run_code('foo=1:100;') + +mlab.stop()