| #!/usr/bin/env python |
| # |
| # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) |
| # Copyright (c) 2008-2016 California Institute of Technology. |
| # Copyright (c) 2016-2022 The Uncertainty Quantification Foundation. |
| # License: 3-clause BSD. The full license text is available at: |
| # - https://github.com/uqfoundation/dill/blob/master/LICENSE |
| # |
| # inspired by inspect.py from Python-2.7.6 |
| # inspect.py author: 'Ka-Ping Yee <ping@lfw.org>' |
| # inspect.py merged into original dill.source by Mike McKerns 4/13/14 |
| """ |
| Extensions to python's 'inspect' module, which can be used |
| to retrieve information from live python objects. The methods |
| defined in this module are augmented to facilitate access to |
| source code of interactively defined functions and classes, |
| as well as provide access to source code for objects defined |
| in a file. |
| """ |
| |
| __all__ = ['findsource', 'getsourcelines', 'getsource', 'indent', 'outdent', \ |
| '_wrap', 'dumpsource', 'getname', '_namespace', 'getimport', \ |
| '_importable', 'importable','isdynamic', 'isfrommain'] |
| |
| import linecache |
| import re |
| from inspect import (getblock, getfile, getmodule, getsourcefile, indentsize, |
| isbuiltin, isclass, iscode, isframe, isfunction, ismethod, |
| ismodule, istraceback) |
| from tokenize import TokenError |
| |
| from ._dill import IS_IPYTHON |
| |
| |
| def isfrommain(obj): |
| "check if object was built in __main__" |
| module = getmodule(obj) |
| if module and module.__name__ == '__main__': |
| return True |
| return False |
| |
| |
| def isdynamic(obj): |
| "check if object was built in the interpreter" |
| try: file = getfile(obj) |
| except TypeError: file = None |
| if file == '<stdin>' and isfrommain(obj): |
| return True |
| return False |
| |
| |
| def _matchlambda(func, line): |
| """check if lambda object 'func' matches raw line of code 'line'""" |
| from .detect import code as getcode |
| from .detect import freevars, globalvars, varnames |
| dummy = lambda : '__this_is_a_big_dummy_function__' |
| # process the line (removing leading whitespace, etc) |
| lhs,rhs = line.split('lambda ',1)[-1].split(":", 1) #FIXME: if !1 inputs |
| try: #FIXME: unsafe |
| _ = eval("lambda %s : %s" % (lhs,rhs), globals(),locals()) |
| except Exception: _ = dummy |
| # get code objects, for comparison |
| _, code = getcode(_).co_code, getcode(func).co_code |
| # check if func is in closure |
| _f = [line.count(i) for i in freevars(func).keys()] |
| if not _f: # not in closure |
| # check if code matches |
| if _ == code: return True |
| return False |
| # weak check on freevars |
| if not all(_f): return False #XXX: VERY WEAK |
| # weak check on varnames and globalvars |
| _f = varnames(func) |
| _f = [line.count(i) for i in _f[0]+_f[1]] |
| if _f and not all(_f): return False #XXX: VERY WEAK |
| _f = [line.count(i) for i in globalvars(func).keys()] |
| if _f and not all(_f): return False #XXX: VERY WEAK |
| # check if func is a double lambda |
| if (line.count('lambda ') > 1) and (lhs in freevars(func).keys()): |
| _lhs,_rhs = rhs.split('lambda ',1)[-1].split(":",1) #FIXME: if !1 inputs |
| try: #FIXME: unsafe |
| _f = eval("lambda %s : %s" % (_lhs,_rhs), globals(),locals()) |
| except Exception: _f = dummy |
| # get code objects, for comparison |
| _, code = getcode(_f).co_code, getcode(func).co_code |
| if len(_) != len(code): return False |
| #NOTE: should be same code same order, but except for 't' and '\x88' |
| _ = set((i,j) for (i,j) in zip(_,code) if i != j) |
| if len(_) != 1: return False #('t','\x88') |
| return True |
| # check indentsize |
| if not indentsize(line): return False #FIXME: is this a good check??? |
| # check if code 'pattern' matches |
| #XXX: or pattern match against dis.dis(code)? (or use uncompyle2?) |
| _ = _.split(_[0]) # 't' #XXX: remove matching values if starts the same? |
| _f = code.split(code[0]) # '\x88' |
| #NOTE: should be same code different order, with different first element |
| _ = dict(re.match(r'([\W\D\S])(.*)', _[i]).groups() for i in range(1,len(_))) |
| _f = dict(re.match(r'([\W\D\S])(.*)', _f[i]).groups() for i in range(1,len(_f))) |
| if (_.keys() == _f.keys()) and (sorted(_.values()) == sorted(_f.values())): |
| return True |
| return False |
| |
| |
| def findsource(object): |
| """Return the entire source file and starting line number for an object. |
| For interactively-defined objects, the 'file' is the interpreter's history. |
| |
| The argument may be a module, class, method, function, traceback, frame, |
| or code object. The source code is returned as a list of all the lines |
| in the file and the line number indexes a line in that list. An IOError |
| is raised if the source code cannot be retrieved, while a TypeError is |
| raised for objects where the source code is unavailable (e.g. builtins).""" |
| |
| module = getmodule(object) |
| try: file = getfile(module) |
| except TypeError: file = None |
| is_module_main = (module and module.__name__ == '__main__' and not file) |
| if IS_IPYTHON and is_module_main: |
| #FIXME: quick fix for functions and classes in IPython interpreter |
| try: |
| file = getfile(object) |
| sourcefile = getsourcefile(object) |
| except TypeError: |
| if isclass(object): |
| for object_method in filter(isfunction, object.__dict__.values()): |
| # look for a method of the class |
| file_candidate = getfile(object_method) |
| if not file_candidate.startswith('<ipython-input-'): |
| continue |
| file = file_candidate |
| sourcefile = getsourcefile(object_method) |
| break |
| if file: |
| lines = linecache.getlines(file) |
| else: |
| # fallback to use history |
| history = '\n'.join(get_ipython().history_manager.input_hist_parsed) |
| lines = [line + '\n' for line in history.splitlines()] |
| # use readline when working in interpreter (i.e. __main__ and not file) |
| elif is_module_main: |
| try: |
| import readline |
| err = '' |
| except ImportError: |
| import sys |
| err = sys.exc_info()[1].args[0] |
| if sys.platform[:3] == 'win': |
| err += ", please install 'pyreadline'" |
| if err: |
| raise IOError(err) |
| lbuf = readline.get_current_history_length() |
| lines = [readline.get_history_item(i)+'\n' for i in range(1,lbuf)] |
| else: |
| try: # special handling for class instances |
| if not isclass(object) and isclass(type(object)): # __class__ |
| file = getfile(module) |
| sourcefile = getsourcefile(module) |
| else: # builtins fail with a TypeError |
| file = getfile(object) |
| sourcefile = getsourcefile(object) |
| except (TypeError, AttributeError): # fail with better error |
| file = getfile(object) |
| sourcefile = getsourcefile(object) |
| if not sourcefile and file[:1] + file[-1:] != '<>': |
| raise IOError('source code not available') |
| file = sourcefile if sourcefile else file |
| |
| module = getmodule(object, file) |
| if module: |
| lines = linecache.getlines(file, module.__dict__) |
| else: |
| lines = linecache.getlines(file) |
| |
| if not lines: |
| raise IOError('could not extract source code') |
| |
| #FIXME: all below may fail if exec used (i.e. exec('f = lambda x:x') ) |
| if ismodule(object): |
| return lines, 0 |
| |
| #NOTE: beneficial if search goes from end to start of buffer history |
| name = pat1 = obj = '' |
| pat2 = r'^(\s*@)' |
| # pat1b = r'^(\s*%s\W*=)' % name #FIXME: finds 'f = decorate(f)', not exec |
| if ismethod(object): |
| name = object.__name__ |
| if name == '<lambda>': pat1 = r'(.*(?<!\w)lambda(:|\s))' |
| else: pat1 = r'^(\s*def\s)' |
| object = object.__func__ |
| if isfunction(object): |
| name = object.__name__ |
| if name == '<lambda>': |
| pat1 = r'(.*(?<!\w)lambda(:|\s))' |
| obj = object #XXX: better a copy? |
| else: pat1 = r'^(\s*def\s)' |
| object = object.__code__ |
| if istraceback(object): |
| object = object.tb_frame |
| if isframe(object): |
| object = object.f_code |
| if iscode(object): |
| if not hasattr(object, 'co_firstlineno'): |
| raise IOError('could not find function definition') |
| stdin = object.co_filename == '<stdin>' |
| if stdin: |
| lnum = len(lines) - 1 # can't get lnum easily, so leverage pat |
| if not pat1: pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)' |
| else: |
| lnum = object.co_firstlineno - 1 |
| pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)' |
| pat1 = re.compile(pat1); pat2 = re.compile(pat2) |
| #XXX: candidate_lnum = [n for n in range(lnum) if pat1.match(lines[n])] |
| while lnum > 0: #XXX: won't find decorators in <stdin> ? |
| line = lines[lnum] |
| if pat1.match(line): |
| if not stdin: break # co_firstlineno does the job |
| if name == '<lambda>': # hackery needed to confirm a match |
| if _matchlambda(obj, line): break |
| else: # not a lambda, just look for the name |
| if name in line: # need to check for decorator... |
| hats = 0 |
| for _lnum in range(lnum-1,-1,-1): |
| if pat2.match(lines[_lnum]): hats += 1 |
| else: break |
| lnum = lnum - hats |
| break |
| lnum = lnum - 1 |
| return lines, lnum |
| |
| try: # turn instances into classes |
| if not isclass(object) and isclass(type(object)): # __class__ |
| object = object.__class__ #XXX: sometimes type(class) is better? |
| #XXX: we don't find how the instance was built |
| except AttributeError: pass |
| if isclass(object): |
| name = object.__name__ |
| pat = re.compile(r'^(\s*)class\s*' + name + r'\b') |
| # make some effort to find the best matching class definition: |
| # use the one with the least indentation, which is the one |
| # that's most probably not inside a function definition. |
| candidates = [] |
| for i in range(len(lines)-1,-1,-1): |
| match = pat.match(lines[i]) |
| if match: |
| # if it's at toplevel, it's already the best one |
| if lines[i][0] == 'c': |
| return lines, i |
| # else add whitespace to candidate list |
| candidates.append((match.group(1), i)) |
| if candidates: |
| # this will sort by whitespace, and by line number, |
| # less whitespace first #XXX: should sort high lnum before low |
| candidates.sort() |
| return lines, candidates[0][1] |
| else: |
| raise IOError('could not find class definition') |
| raise IOError('could not find code object') |
| |
| |
| def getblocks(object, lstrip=False, enclosing=False, locate=False): |
| """Return a list of source lines and starting line number for an object. |
| Interactively-defined objects refer to lines in the interpreter's history. |
| |
| If enclosing=True, then also return any enclosing code. |
| If lstrip=True, ensure there is no indentation in the first line of code. |
| If locate=True, then also return the line number for the block of code. |
| |
| DEPRECATED: use 'getsourcelines' instead |
| """ |
| lines, lnum = findsource(object) |
| |
| if ismodule(object): |
| if lstrip: lines = _outdent(lines) |
| return ([lines], [0]) if locate is True else [lines] |
| |
| #XXX: 'enclosing' means: closures only? or classes and files? |
| indent = indentsize(lines[lnum]) |
| block = getblock(lines[lnum:]) #XXX: catch any TokenError here? |
| |
| if not enclosing or not indent: |
| if lstrip: block = _outdent(block) |
| return ([block], [lnum]) if locate is True else [block] |
| |
| pat1 = r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))'; pat1 = re.compile(pat1) |
| pat2 = r'^(\s*@)'; pat2 = re.compile(pat2) |
| #pat3 = r'^(\s*class\s)'; pat3 = re.compile(pat3) #XXX: enclosing class? |
| #FIXME: bound methods need enclosing class (and then instantiation) |
| # *or* somehow apply a partial using the instance |
| |
| skip = 0 |
| line = 0 |
| blocks = []; _lnum = [] |
| target = ''.join(block) |
| while line <= lnum: #XXX: repeat lnum? or until line < lnum? |
| # see if starts with ('def','lambda') and contains our target block |
| if pat1.match(lines[line]): |
| if not skip: |
| try: code = getblock(lines[line:]) |
| except TokenError: code = [lines[line]] |
| if indentsize(lines[line]) > indent: #XXX: should be >= ? |
| line += len(code) - skip |
| elif target in ''.join(code): |
| blocks.append(code) # save code block as the potential winner |
| _lnum.append(line - skip) # save the line number for the match |
| line += len(code) - skip |
| else: |
| line += 1 |
| skip = 0 |
| # find skip: the number of consecutive decorators |
| elif pat2.match(lines[line]): |
| try: code = getblock(lines[line:]) |
| except TokenError: code = [lines[line]] |
| skip = 1 |
| for _line in code[1:]: # skip lines that are decorators |
| if not pat2.match(_line): break |
| skip += 1 |
| line += skip |
| # no match: reset skip and go to the next line |
| else: |
| line +=1 |
| skip = 0 |
| |
| if not blocks: |
| blocks = [block] |
| _lnum = [lnum] |
| if lstrip: blocks = [_outdent(block) for block in blocks] |
| # return last match |
| return (blocks, _lnum) if locate is True else blocks |
| |
| |
| def getsourcelines(object, lstrip=False, enclosing=False): |
| """Return a list of source lines and starting line number for an object. |
| Interactively-defined objects refer to lines in the interpreter's history. |
| |
| The argument may be a module, class, method, function, traceback, frame, |
| or code object. The source code is returned as a list of the lines |
| corresponding to the object and the line number indicates where in the |
| original source file the first line of code was found. An IOError is |
| raised if the source code cannot be retrieved, while a TypeError is |
| raised for objects where the source code is unavailable (e.g. builtins). |
| |
| If lstrip=True, ensure there is no indentation in the first line of code. |
| If enclosing=True, then also return any enclosing code.""" |
| code, n = getblocks(object, lstrip=lstrip, enclosing=enclosing, locate=True) |
| return code[-1], n[-1] |
| |
| |
| #NOTE: broke backward compatibility 4/16/14 (was lstrip=True, force=True) |
| def getsource(object, alias='', lstrip=False, enclosing=False, \ |
| force=False, builtin=False): |
| """Return the text of the source code for an object. The source code for |
| interactively-defined objects are extracted from the interpreter's history. |
| |
| The argument may be a module, class, method, function, traceback, frame, |
| or code object. The source code is returned as a single string. An |
| IOError is raised if the source code cannot be retrieved, while a |
| TypeError is raised for objects where the source code is unavailable |
| (e.g. builtins). |
| |
| If alias is provided, then add a line of code that renames the object. |
| If lstrip=True, ensure there is no indentation in the first line of code. |
| If enclosing=True, then also return any enclosing code. |
| If force=True, catch (TypeError,IOError) and try to use import hooks. |
| If builtin=True, force an import for any builtins |
| """ |
| # hascode denotes a callable |
| hascode = _hascode(object) |
| # is a class instance type (and not in builtins) |
| instance = _isinstance(object) |
| |
| # get source lines; if fail, try to 'force' an import |
| try: # fails for builtins, and other assorted object types |
| lines, lnum = getsourcelines(object, enclosing=enclosing) |
| except (TypeError, IOError): # failed to get source, resort to import hooks |
| if not force: # don't try to get types that findsource can't get |
| raise |
| if not getmodule(object): # get things like 'None' and '1' |
| if not instance: return getimport(object, alias, builtin=builtin) |
| # special handling (numpy arrays, ...) |
| _import = getimport(object, builtin=builtin) |
| name = getname(object, force=True) |
| _alias = "%s = " % alias if alias else "" |
| if alias == name: _alias = "" |
| return _import+_alias+"%s\n" % name |
| else: #FIXME: could use a good bit of cleanup, since using getimport... |
| if not instance: return getimport(object, alias, builtin=builtin) |
| # now we are dealing with an instance... |
| name = object.__class__.__name__ |
| module = object.__module__ |
| if module in ['builtins','__builtin__']: |
| return getimport(object, alias, builtin=builtin) |
| else: #FIXME: leverage getimport? use 'from module import name'? |
| lines, lnum = ["%s = __import__('%s', fromlist=['%s']).%s\n" % (name,module,name,name)], 0 |
| obj = eval(lines[0].lstrip(name + ' = ')) |
| lines, lnum = getsourcelines(obj, enclosing=enclosing) |
| |
| # strip leading indent (helps ensure can be imported) |
| if lstrip or alias: |
| lines = _outdent(lines) |
| |
| # instantiate, if there's a nice repr #XXX: BAD IDEA??? |
| if instance: #and force: #XXX: move into findsource or getsourcelines ? |
| if '(' in repr(object): lines.append('%r\n' % object) |
| #else: #XXX: better to somehow to leverage __reduce__ ? |
| # reconstructor,args = object.__reduce__() |
| # _ = reconstructor(*args) |
| else: # fall back to serialization #XXX: bad idea? |
| #XXX: better not duplicate work? #XXX: better new/enclose=True? |
| lines = dumpsource(object, alias='', new=force, enclose=False) |
| lines, lnum = [line+'\n' for line in lines.split('\n')][:-1], 0 |
| #else: object.__code__ # raise AttributeError |
| |
| # add an alias to the source code |
| if alias: |
| if hascode: |
| skip = 0 |
| for line in lines: # skip lines that are decorators |
| if not line.startswith('@'): break |
| skip += 1 |
| #XXX: use regex from findsource / getsourcelines ? |
| if lines[skip].lstrip().startswith('def '): # we have a function |
| if alias != object.__name__: |
| lines.append('\n%s = %s\n' % (alias, object.__name__)) |
| elif 'lambda ' in lines[skip]: # we have a lambda |
| if alias != lines[skip].split('=')[0].strip(): |
| lines[skip] = '%s = %s' % (alias, lines[skip]) |
| else: # ...try to use the object's name |
| if alias != object.__name__: |
| lines.append('\n%s = %s\n' % (alias, object.__name__)) |
| else: # class or class instance |
| if instance: |
| if alias != lines[-1].split('=')[0].strip(): |
| lines[-1] = ('%s = ' % alias) + lines[-1] |
| else: |
| name = getname(object, force=True) or object.__name__ |
| if alias != name: |
| lines.append('\n%s = %s\n' % (alias, name)) |
| return ''.join(lines) |
| |
| |
| def _hascode(object): |
| '''True if object has an attribute that stores it's __code__''' |
| return getattr(object,'__code__',None) or getattr(object,'func_code',None) |
| |
| def _isinstance(object): |
| '''True if object is a class instance type (and is not a builtin)''' |
| if _hascode(object) or isclass(object) or ismodule(object): |
| return False |
| if istraceback(object) or isframe(object) or iscode(object): |
| return False |
| # special handling (numpy arrays, ...) |
| if not getmodule(object) and getmodule(type(object)).__name__ in ['numpy']: |
| return True |
| # # check if is instance of a builtin |
| # if not getmodule(object) and getmodule(type(object)).__name__ in ['__builtin__','builtins']: |
| # return False |
| _types = ('<class ',"<type 'instance'>") |
| if not repr(type(object)).startswith(_types): #FIXME: weak hack |
| return False |
| if not getmodule(object) or object.__module__ in ['builtins','__builtin__'] or getname(object, force=True) in ['array']: |
| return False |
| return True # by process of elimination... it's what we want |
| |
| |
| def _intypes(object): |
| '''check if object is in the 'types' module''' |
| import types |
| # allow user to pass in object or object.__name__ |
| if type(object) is not type(''): |
| object = getname(object, force=True) |
| if object == 'ellipsis': object = 'EllipsisType' |
| return True if hasattr(types, object) else False |
| |
| |
| def _isstring(object): #XXX: isstringlike better? |
| '''check if object is a string-like type''' |
| return isinstance(object, (str, bytes)) |
| |
| |
| def indent(code, spaces=4): |
| '''indent a block of code with whitespace (default is 4 spaces)''' |
| indent = indentsize(code) |
| if type(spaces) is int: spaces = ' '*spaces |
| # if '\t' is provided, will indent with a tab |
| nspaces = indentsize(spaces) |
| # blank lines (etc) need to be ignored |
| lines = code.split('\n') |
| ## stq = "'''"; dtq = '"""' |
| ## in_stq = in_dtq = False |
| for i in range(len(lines)): |
| #FIXME: works... but shouldn't indent 2nd+ lines of multiline doc |
| _indent = indentsize(lines[i]) |
| if indent > _indent: continue |
| lines[i] = spaces+lines[i] |
| ## #FIXME: may fail when stq and dtq in same line (depends on ordering) |
| ## nstq, ndtq = lines[i].count(stq), lines[i].count(dtq) |
| ## if not in_dtq and not in_stq: |
| ## lines[i] = spaces+lines[i] # we indent |
| ## # entering a comment block |
| ## if nstq%2: in_stq = not in_stq |
| ## if ndtq%2: in_dtq = not in_dtq |
| ## # leaving a comment block |
| ## elif in_dtq and ndtq%2: in_dtq = not in_dtq |
| ## elif in_stq and nstq%2: in_stq = not in_stq |
| ## else: pass |
| if lines[-1].strip() == '': lines[-1] = '' |
| return '\n'.join(lines) |
| |
| |
| def _outdent(lines, spaces=None, all=True): |
| '''outdent lines of code, accounting for docs and line continuations''' |
| indent = indentsize(lines[0]) |
| if spaces is None or spaces > indent or spaces < 0: spaces = indent |
| for i in range(len(lines) if all else 1): |
| #FIXME: works... but shouldn't outdent 2nd+ lines of multiline doc |
| _indent = indentsize(lines[i]) |
| if spaces > _indent: _spaces = _indent |
| else: _spaces = spaces |
| lines[i] = lines[i][_spaces:] |
| return lines |
| |
| def outdent(code, spaces=None, all=True): |
| '''outdent a block of code (default is to strip all leading whitespace)''' |
| indent = indentsize(code) |
| if spaces is None or spaces > indent or spaces < 0: spaces = indent |
| #XXX: will this delete '\n' in some cases? |
| if not all: return code[spaces:] |
| return '\n'.join(_outdent(code.split('\n'), spaces=spaces, all=all)) |
| |
| |
| #XXX: not sure what the point of _wrap is... |
| __globals__ = globals() |
| __locals__ = locals() |
| def _wrap(f): |
| """ encapsulate a function and it's __import__ """ |
| def func(*args, **kwds): |
| try: |
| # _ = eval(getsource(f, force=True)) #XXX: safer but less robust |
| exec(getimportable(f, alias='_'), __globals__, __locals__) |
| except Exception: |
| raise ImportError('cannot import name ' + f.__name__) |
| return _(*args, **kwds) |
| func.__name__ = f.__name__ |
| func.__doc__ = f.__doc__ |
| return func |
| |
| |
| def _enclose(object, alias=''): #FIXME: needs alias to hold returned object |
| """create a function enclosure around the source of some object""" |
| #XXX: dummy and stub should append a random string |
| dummy = '__this_is_a_big_dummy_enclosing_function__' |
| stub = '__this_is_a_stub_variable__' |
| code = 'def %s():\n' % dummy |
| code += indent(getsource(object, alias=stub, lstrip=True, force=True)) |
| code += indent('return %s\n' % stub) |
| if alias: code += '%s = ' % alias |
| code += '%s(); del %s\n' % (dummy, dummy) |
| #code += "globals().pop('%s',lambda :None)()\n" % dummy |
| return code |
| |
| |
| def dumpsource(object, alias='', new=False, enclose=True): |
| """'dump to source', where the code includes a pickled object. |
| |
| If new=True and object is a class instance, then create a new |
| instance using the unpacked class source code. If enclose, then |
| create the object inside a function enclosure (thus minimizing |
| any global namespace pollution). |
| """ |
| from dill import dumps |
| pik = repr(dumps(object)) |
| code = 'import dill\n' |
| if enclose: |
| stub = '__this_is_a_stub_variable__' #XXX: *must* be same _enclose.stub |
| pre = '%s = ' % stub |
| new = False #FIXME: new=True doesn't work with enclose=True |
| else: |
| stub = alias |
| pre = '%s = ' % stub if alias else alias |
| |
| # if a 'new' instance is not needed, then just dump and load |
| if not new or not _isinstance(object): |
| code += pre + 'dill.loads(%s)\n' % pik |
| else: #XXX: other cases where source code is needed??? |
| code += getsource(object.__class__, alias='', lstrip=True, force=True) |
| mod = repr(object.__module__) # should have a module (no builtins here) |
| code += pre + 'dill.loads(%s.replace(b%s,bytes(__name__,"UTF-8")))\n' % (pik,mod) |
| #code += 'del %s' % object.__class__.__name__ #NOTE: kills any existing! |
| |
| if enclose: |
| # generation of the 'enclosure' |
| dummy = '__this_is_a_big_dummy_object__' |
| dummy = _enclose(dummy, alias=alias) |
| # hack to replace the 'dummy' with the 'real' code |
| dummy = dummy.split('\n') |
| code = dummy[0]+'\n' + indent(code) + '\n'.join(dummy[-3:]) |
| |
| return code #XXX: better 'dumpsourcelines', returning list of lines? |
| |
| |
| def getname(obj, force=False, fqn=False): #XXX: throw(?) to raise error on fail? |
| """get the name of the object. for lambdas, get the name of the pointer """ |
| if fqn: return '.'.join(_namespace(obj)) |
| module = getmodule(obj) |
| if not module: # things like "None" and "1" |
| if not force: return None |
| return repr(obj) |
| try: |
| #XXX: 'wrong' for decorators and curried functions ? |
| # if obj.func_closure: ...use logic from getimportable, etc ? |
| name = obj.__name__ |
| if name == '<lambda>': |
| return getsource(obj).split('=',1)[0].strip() |
| # handle some special cases |
| if module.__name__ in ['builtins','__builtin__']: |
| if name == 'ellipsis': name = 'EllipsisType' |
| return name |
| except AttributeError: #XXX: better to just throw AttributeError ? |
| if not force: return None |
| name = repr(obj) |
| if name.startswith('<'): # or name.split('('): |
| return None |
| return name |
| |
| |
| def _namespace(obj): |
| """_namespace(obj); return namespace hierarchy (as a list of names) |
| for the given object. For an instance, find the class hierarchy. |
| |
| For example: |
| |
| >>> from functools import partial |
| >>> p = partial(int, base=2) |
| >>> _namespace(p) |
| [\'functools\', \'partial\'] |
| """ |
| # mostly for functions and modules and such |
| #FIXME: 'wrong' for decorators and curried functions |
| try: #XXX: needs some work and testing on different types |
| module = qual = str(getmodule(obj)).split()[1].strip('>').strip('"').strip("'") |
| qual = qual.split('.') |
| if ismodule(obj): |
| return qual |
| # get name of a lambda, function, etc |
| name = getname(obj) or obj.__name__ # failing, raise AttributeError |
| # check special cases (NoneType, ...) |
| if module in ['builtins','__builtin__']: # BuiltinFunctionType |
| if _intypes(name): return ['types'] + [name] |
| return qual + [name] #XXX: can be wrong for some aliased objects |
| except Exception: pass |
| # special case: numpy.inf and numpy.nan (we don't want them as floats) |
| if str(obj) in ['inf','nan','Inf','NaN']: # is more, but are they needed? |
| return ['numpy'] + [str(obj)] |
| # mostly for classes and class instances and such |
| module = getattr(obj.__class__, '__module__', None) |
| qual = str(obj.__class__) |
| try: qual = qual[qual.index("'")+1:-2] |
| except ValueError: pass # str(obj.__class__) made the 'try' unnecessary |
| qual = qual.split(".") |
| if module in ['builtins','__builtin__']: |
| # check special cases (NoneType, Ellipsis, ...) |
| if qual[-1] == 'ellipsis': qual[-1] = 'EllipsisType' |
| if _intypes(qual[-1]): module = 'types' #XXX: BuiltinFunctionType |
| qual = [module] + qual |
| return qual |
| |
| |
| #NOTE: 05/25/14 broke backward compatability: added 'alias' as 3rd argument |
| def _getimport(head, tail, alias='', verify=True, builtin=False): |
| """helper to build a likely import string from head and tail of namespace. |
| ('head','tail') are used in the following context: "from head import tail" |
| |
| If verify=True, then test the import string before returning it. |
| If builtin=True, then force an import for builtins where possible. |
| If alias is provided, then rename the object on import. |
| """ |
| # special handling for a few common types |
| if tail in ['Ellipsis', 'NotImplemented'] and head in ['types']: |
| head = len.__module__ |
| elif tail in ['None'] and head in ['types']: |
| _alias = '%s = ' % alias if alias else '' |
| if alias == tail: _alias = '' |
| return _alias+'%s\n' % tail |
| # we don't need to import from builtins, so return '' |
| # elif tail in ['NoneType','int','float','long','complex']: return '' #XXX: ? |
| if head in ['builtins','__builtin__']: |
| # special cases (NoneType, Ellipsis, ...) #XXX: BuiltinFunctionType |
| if tail == 'ellipsis': tail = 'EllipsisType' |
| if _intypes(tail): head = 'types' |
| elif not builtin: |
| _alias = '%s = ' % alias if alias else '' |
| if alias == tail: _alias = '' |
| return _alias+'%s\n' % tail |
| else: pass # handle builtins below |
| # get likely import string |
| if not head: _str = "import %s" % tail |
| else: _str = "from %s import %s" % (head, tail) |
| _alias = " as %s\n" % alias if alias else "\n" |
| if alias == tail: _alias = "\n" |
| _str += _alias |
| # FIXME: fails on most decorators, currying, and such... |
| # (could look for magic __wrapped__ or __func__ attr) |
| # (could fix in 'namespace' to check obj for closure) |
| if verify and not head.startswith('dill.'):# weird behavior for dill |
| #print(_str) |
| try: exec(_str) #XXX: check if == obj? (name collision) |
| except ImportError: #XXX: better top-down or bottom-up recursion? |
| _head = head.rsplit(".",1)[0] #(or get all, then compare == obj?) |
| if not _head: raise |
| if _head != head: |
| _str = _getimport(_head, tail, alias, verify) |
| return _str |
| |
| |
| #XXX: rename builtin to force? vice versa? verify to force? (as in getsource) |
| #NOTE: 05/25/14 broke backward compatability: added 'alias' as 2nd argument |
| def getimport(obj, alias='', verify=True, builtin=False, enclosing=False): |
| """get the likely import string for the given object |
| |
| obj is the object to inspect |
| If verify=True, then test the import string before returning it. |
| If builtin=True, then force an import for builtins where possible. |
| If enclosing=True, get the import for the outermost enclosing callable. |
| If alias is provided, then rename the object on import. |
| """ |
| if enclosing: |
| from .detect import outermost |
| _obj = outermost(obj) |
| obj = _obj if _obj else obj |
| # get the namespace |
| qual = _namespace(obj) |
| head = '.'.join(qual[:-1]) |
| tail = qual[-1] |
| # for named things... with a nice repr #XXX: move into _namespace? |
| try: # look for '<...>' and be mindful it might be in lists, dicts, etc... |
| name = repr(obj).split('<',1)[1].split('>',1)[1] |
| name = None # we have a 'object'-style repr |
| except Exception: # it's probably something 'importable' |
| if head in ['builtins','__builtin__']: |
| name = repr(obj) #XXX: catch [1,2], (1,2), set([1,2])... others? |
| else: |
| name = repr(obj).split('(')[0] |
| #if not repr(obj).startswith('<'): name = repr(obj).split('(')[0] |
| #else: name = None |
| if name: # try using name instead of tail |
| try: return _getimport(head, name, alias, verify, builtin) |
| except ImportError: pass |
| except SyntaxError: |
| if head in ['builtins','__builtin__']: |
| _alias = '%s = ' % alias if alias else '' |
| if alias == name: _alias = '' |
| return _alias+'%s\n' % name |
| else: pass |
| try: |
| #if type(obj) is type(abs): _builtin = builtin # BuiltinFunctionType |
| #else: _builtin = False |
| return _getimport(head, tail, alias, verify, builtin) |
| except ImportError: |
| raise # could do some checking against obj |
| except SyntaxError: |
| if head in ['builtins','__builtin__']: |
| _alias = '%s = ' % alias if alias else '' |
| if alias == tail: _alias = '' |
| return _alias+'%s\n' % tail |
| raise # could do some checking against obj |
| |
| |
| def _importable(obj, alias='', source=None, enclosing=False, force=True, \ |
| builtin=True, lstrip=True): |
| """get an import string (or the source code) for the given object |
| |
| This function will attempt to discover the name of the object, or the repr |
| of the object, or the source code for the object. To attempt to force |
| discovery of the source code, use source=True, to attempt to force the |
| use of an import, use source=False; otherwise an import will be sought |
| for objects not defined in __main__. The intent is to build a string |
| that can be imported from a python file. obj is the object to inspect. |
| If alias is provided, then rename the object with the given alias. |
| |
| If source=True, use these options: |
| If enclosing=True, then also return any enclosing code. |
| If force=True, catch (TypeError,IOError) and try to use import hooks. |
| If lstrip=True, ensure there is no indentation in the first line of code. |
| |
| If source=False, use these options: |
| If enclosing=True, get the import for the outermost enclosing callable. |
| If force=True, then don't test the import string before returning it. |
| If builtin=True, then force an import for builtins where possible. |
| """ |
| if source is None: |
| source = True if isfrommain(obj) else False |
| if source: # first try to get the source |
| try: |
| return getsource(obj, alias, enclosing=enclosing, \ |
| force=force, lstrip=lstrip, builtin=builtin) |
| except Exception: pass |
| try: |
| if not _isinstance(obj): |
| return getimport(obj, alias, enclosing=enclosing, \ |
| verify=(not force), builtin=builtin) |
| # first 'get the import', then 'get the instance' |
| _import = getimport(obj, enclosing=enclosing, \ |
| verify=(not force), builtin=builtin) |
| name = getname(obj, force=True) |
| if not name: |
| raise AttributeError("object has no atribute '__name__'") |
| _alias = "%s = " % alias if alias else "" |
| if alias == name: _alias = "" |
| return _import+_alias+"%s\n" % name |
| |
| except Exception: pass |
| if not source: # try getsource, only if it hasn't been tried yet |
| try: |
| return getsource(obj, alias, enclosing=enclosing, \ |
| force=force, lstrip=lstrip, builtin=builtin) |
| except Exception: pass |
| # get the name (of functions, lambdas, and classes) |
| # or hope that obj can be built from the __repr__ |
| #XXX: what to do about class instances and such? |
| obj = getname(obj, force=force) |
| # we either have __repr__ or __name__ (or None) |
| if not obj or obj.startswith('<'): |
| raise AttributeError("object has no atribute '__name__'") |
| _alias = '%s = ' % alias if alias else '' |
| if alias == obj: _alias = '' |
| return _alias+'%s\n' % obj |
| #XXX: possible failsafe... (for example, for instances when source=False) |
| # "import dill; result = dill.loads(<pickled_object>); # repr(<object>)" |
| |
| def _closuredimport(func, alias='', builtin=False): |
| """get import for closured objects; return a dict of 'name' and 'import'""" |
| import re |
| from .detect import freevars, outermost |
| free_vars = freevars(func) |
| func_vars = {} |
| # split into 'funcs' and 'non-funcs' |
| for name,obj in list(free_vars.items()): |
| if not isfunction(obj): continue |
| # get import for 'funcs' |
| fobj = free_vars.pop(name) |
| src = getsource(fobj) |
| if src.lstrip().startswith('@'): # we have a decorator |
| src = getimport(fobj, alias=alias, builtin=builtin) |
| else: # we have to "hack" a bit... and maybe be lucky |
| encl = outermost(func) |
| # pattern: 'func = enclosing(fobj' |
| pat = r'.*[\w\s]=\s*'+getname(encl)+r'\('+getname(fobj) |
| mod = getname(getmodule(encl)) |
| #HACK: get file containing 'outer' function; is func there? |
| lines,_ = findsource(encl) |
| candidate = [line for line in lines if getname(encl) in line and \ |
| re.match(pat, line)] |
| if not candidate: |
| mod = getname(getmodule(fobj)) |
| #HACK: get file containing 'inner' function; is func there? |
| lines,_ = findsource(fobj) |
| candidate = [line for line in lines \ |
| if getname(fobj) in line and re.match(pat, line)] |
| if not len(candidate): raise TypeError('import could not be found') |
| candidate = candidate[-1] |
| name = candidate.split('=',1)[0].split()[-1].strip() |
| src = _getimport(mod, name, alias=alias, builtin=builtin) |
| func_vars[name] = src |
| if not func_vars: |
| name = outermost(func) |
| mod = getname(getmodule(name)) |
| if not mod or name is func: # then it can be handled by getimport |
| name = getname(func, force=True) #XXX: better key? |
| src = getimport(func, alias=alias, builtin=builtin) |
| else: |
| lines,_ = findsource(name) |
| # pattern: 'func = enclosing(' |
| candidate = [line for line in lines if getname(name) in line and \ |
| re.match(r'.*[\w\s]=\s*'+getname(name)+r'\(', line)] |
| if not len(candidate): raise TypeError('import could not be found') |
| candidate = candidate[-1] |
| name = candidate.split('=',1)[0].split()[-1].strip() |
| src = _getimport(mod, name, alias=alias, builtin=builtin) |
| func_vars[name] = src |
| return func_vars |
| |
| #XXX: should be able to use __qualname__ |
| def _closuredsource(func, alias=''): |
| """get source code for closured objects; return a dict of 'name' |
| and 'code blocks'""" |
| #FIXME: this entire function is a messy messy HACK |
| # - pollutes global namespace |
| # - fails if name of freevars are reused |
| # - can unnecessarily duplicate function code |
| from .detect import freevars |
| free_vars = freevars(func) |
| func_vars = {} |
| # split into 'funcs' and 'non-funcs' |
| for name,obj in list(free_vars.items()): |
| if not isfunction(obj): |
| # get source for 'non-funcs' |
| free_vars[name] = getsource(obj, force=True, alias=name) |
| continue |
| # get source for 'funcs' |
| fobj = free_vars.pop(name) |
| src = getsource(fobj, alias) # DO NOT include dependencies |
| # if source doesn't start with '@', use name as the alias |
| if not src.lstrip().startswith('@'): #FIXME: 'enclose' in dummy; |
| src = importable(fobj,alias=name)# wrong ref 'name' |
| org = getsource(func, alias, enclosing=False, lstrip=True) |
| src = (src, org) # undecorated first, then target |
| else: #NOTE: reproduces the code! |
| org = getsource(func, enclosing=True, lstrip=False) |
| src = importable(fobj, alias, source=True) # include dependencies |
| src = (org, src) # target first, then decorated |
| func_vars[name] = src |
| src = ''.join(free_vars.values()) |
| if not func_vars: #FIXME: 'enclose' in dummy; wrong ref 'name' |
| org = getsource(func, alias, force=True, enclosing=False, lstrip=True) |
| src = (src, org) # variables first, then target |
| else: |
| src = (src, None) # just variables (better '' instead of None?) |
| func_vars[None] = src |
| # FIXME: remove duplicates (however, order is important...) |
| return func_vars |
| |
| def importable(obj, alias='', source=None, builtin=True): |
| """get an importable string (i.e. source code or the import string) |
| for the given object, including any required objects from the enclosing |
| and global scope |
| |
| This function will attempt to discover the name of the object, or the repr |
| of the object, or the source code for the object. To attempt to force |
| discovery of the source code, use source=True, to attempt to force the |
| use of an import, use source=False; otherwise an import will be sought |
| for objects not defined in __main__. The intent is to build a string |
| that can be imported from a python file. |
| |
| obj is the object to inspect. If alias is provided, then rename the |
| object with the given alias. If builtin=True, then force an import for |
| builtins where possible. |
| """ |
| #NOTE: we always 'force', and 'lstrip' as necessary |
| #NOTE: for 'enclosing', use importable(outermost(obj)) |
| if source is None: |
| source = True if isfrommain(obj) else False |
| elif builtin and isbuiltin(obj): |
| source = False |
| tried_source = tried_import = False |
| while True: |
| if not source: # we want an import |
| try: |
| if _isinstance(obj): # for instances, punt to _importable |
| return _importable(obj, alias, source=False, builtin=builtin) |
| src = _closuredimport(obj, alias=alias, builtin=builtin) |
| if len(src) == 0: |
| raise NotImplementedError('not implemented') |
| if len(src) > 1: |
| raise NotImplementedError('not implemented') |
| return list(src.values())[0] |
| except Exception: |
| if tried_source: raise |
| tried_import = True |
| # we want the source |
| try: |
| src = _closuredsource(obj, alias=alias) |
| if len(src) == 0: |
| raise NotImplementedError('not implemented') |
| # groan... an inline code stitcher |
| def _code_stitcher(block): |
| "stitch together the strings in tuple 'block'" |
| if block[0] and block[-1]: block = '\n'.join(block) |
| elif block[0]: block = block[0] |
| elif block[-1]: block = block[-1] |
| else: block = '' |
| return block |
| # get free_vars first |
| _src = _code_stitcher(src.pop(None)) |
| _src = [_src] if _src else [] |
| # get func_vars |
| for xxx in src.values(): |
| xxx = _code_stitcher(xxx) |
| if xxx: _src.append(xxx) |
| # make a single source string |
| if not len(_src): |
| src = '' |
| elif len(_src) == 1: |
| src = _src[0] |
| else: |
| src = '\n'.join(_src) |
| # get source code of objects referred to by obj in global scope |
| from .detect import globalvars |
| obj = globalvars(obj) #XXX: don't worry about alias? recurse? etc? |
| obj = list(getsource(_obj,name,force=True) for (name,_obj) in obj.items() if not isbuiltin(_obj)) |
| obj = '\n'.join(obj) if obj else '' |
| # combine all referred-to source (global then enclosing) |
| if not obj: return src |
| if not src: return obj |
| return obj + src |
| except Exception: |
| if tried_import: raise |
| tried_source = True |
| source = not source |
| # should never get here |
| return |
| |
| |
| # backward compatability |
| def getimportable(obj, alias='', byname=True, explicit=False): |
| return importable(obj,alias,source=(not byname),builtin=explicit) |
| #return outdent(_importable(obj,alias,source=(not byname),builtin=explicit)) |
| def likely_import(obj, passive=False, explicit=False): |
| return getimport(obj, verify=(not passive), builtin=explicit) |
| def _likely_import(first, last, passive=False, explicit=True): |
| return _getimport(first, last, verify=(not passive), builtin=explicit) |
| _get_name = getname |
| getblocks_from_history = getblocks |
| |
| |
| |
| # EOF |
