| #!/usr/bin/env python |
| # |
| # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) |
| # Copyright (c) 2008-2016 California Institute of Technology. |
| # Copyright (c) 2016-2023 The Uncertainty Quantification Foundation. |
| # License: 3-clause BSD. The full license text is available at: |
| # - https://github.com/uqfoundation/dill/blob/master/LICENSE |
| """ |
| Methods for detecting objects leading to pickling failures. |
| """ |
| |
| import dis |
| from inspect import ismethod, isfunction, istraceback, isframe, iscode |
| |
| from .pointers import parent, reference, at, parents, children |
| from .logger import trace |
| |
| __all__ = ['baditems','badobjects','badtypes','code','errors','freevars', |
| 'getmodule','globalvars','nestedcode','nestedglobals','outermost', |
| 'referredglobals','referrednested','trace','varnames'] |
| |
| def getmodule(object, _filename=None, force=False): |
| """get the module of the object""" |
| from inspect import getmodule as getmod |
| module = getmod(object, _filename) |
| if module or not force: return module |
| import builtins |
| from .source import getname |
| name = getname(object, force=True) |
| return builtins if name in vars(builtins).keys() else None |
| |
| def outermost(func): # is analogous to getsource(func,enclosing=True) |
| """get outermost enclosing object (i.e. the outer function in a closure) |
| |
| NOTE: this is the object-equivalent of getsource(func, enclosing=True) |
| """ |
| if ismethod(func): |
| _globals = func.__func__.__globals__ or {} |
| elif isfunction(func): |
| _globals = func.__globals__ or {} |
| else: |
| return #XXX: or raise? no matches |
| _globals = _globals.items() |
| # get the enclosing source |
| from .source import getsourcelines |
| try: lines,lnum = getsourcelines(func, enclosing=True) |
| except Exception: #TypeError, IOError |
| lines,lnum = [],None |
| code = ''.join(lines) |
| # get all possible names,objects that are named in the enclosing source |
| _locals = ((name,obj) for (name,obj) in _globals if name in code) |
| # now only save the objects that generate the enclosing block |
| for name,obj in _locals: #XXX: don't really need 'name' |
| try: |
| if getsourcelines(obj) == (lines,lnum): return obj |
| except Exception: #TypeError, IOError |
| pass |
| return #XXX: or raise? no matches |
| |
| def nestedcode(func, recurse=True): #XXX: or return dict of {co_name: co} ? |
| """get the code objects for any nested functions (e.g. in a closure)""" |
| func = code(func) |
| if not iscode(func): return [] #XXX: or raise? no matches |
| nested = set() |
| for co in func.co_consts: |
| if co is None: continue |
| co = code(co) |
| if co: |
| nested.add(co) |
| if recurse: nested |= set(nestedcode(co, recurse=True)) |
| return list(nested) |
| |
| def code(func): |
| """get the code object for the given function or method |
| |
| NOTE: use dill.source.getsource(CODEOBJ) to get the source code |
| """ |
| if ismethod(func): func = func.__func__ |
| if isfunction(func): func = func.__code__ |
| if istraceback(func): func = func.tb_frame |
| if isframe(func): func = func.f_code |
| if iscode(func): return func |
| return |
| |
| #XXX: ugly: parse dis.dis for name after "<code object" in line and in globals? |
| def referrednested(func, recurse=True): #XXX: return dict of {__name__: obj} ? |
| """get functions defined inside of func (e.g. inner functions in a closure) |
| |
| NOTE: results may differ if the function has been executed or not. |
| If len(nestedcode(func)) > len(referrednested(func)), try calling func(). |
| If possible, python builds code objects, but delays building functions |
| until func() is called. |
| """ |
| import gc |
| funcs = set() |
| # get the code objects, and try to track down by referrence |
| for co in nestedcode(func, recurse): |
| # look for function objects that refer to the code object |
| for obj in gc.get_referrers(co): |
| # get methods |
| _ = getattr(obj, '__func__', None) # ismethod |
| if getattr(_, '__code__', None) is co: funcs.add(obj) |
| # get functions |
| elif getattr(obj, '__code__', None) is co: funcs.add(obj) |
| # get frame objects |
| elif getattr(obj, 'f_code', None) is co: funcs.add(obj) |
| # get code objects |
| elif hasattr(obj, 'co_code') and obj is co: funcs.add(obj) |
| # frameobjs => func.__code__.co_varnames not in func.__code__.co_cellvars |
| # funcobjs => func.__code__.co_cellvars not in func.__code__.co_varnames |
| # frameobjs are not found, however funcobjs are... |
| # (see: test_mixins.quad ... and test_mixins.wtf) |
| # after execution, code objects get compiled, and then may be found by gc |
| return list(funcs) |
| |
| |
| def freevars(func): |
| """get objects defined in enclosing code that are referred to by func |
| |
| returns a dict of {name:object}""" |
| if ismethod(func): func = func.__func__ |
| if isfunction(func): |
| closures = func.__closure__ or () |
| func = func.__code__.co_freevars # get freevars |
| else: |
| return {} |
| |
| def get_cell_contents(): |
| for name, c in zip(func, closures): |
| try: |
| cell_contents = c.cell_contents |
| except ValueError: # cell is empty |
| continue |
| yield name, c.cell_contents |
| |
| return dict(get_cell_contents()) |
| |
| # thanks to Davies Liu for recursion of globals |
| def nestedglobals(func, recurse=True): |
| """get the names of any globals found within func""" |
| func = code(func) |
| if func is None: return list() |
| import sys |
| from .temp import capture |
| CAN_NULL = sys.hexversion >= 0x30b00a7 # NULL may be prepended >= 3.11a7 |
| names = set() |
| with capture('stdout') as out: |
| dis.dis(func) #XXX: dis.dis(None) disassembles last traceback |
| for line in out.getvalue().splitlines(): |
| if '_GLOBAL' in line: |
| name = line.split('(')[-1].split(')')[0] |
| if CAN_NULL: |
| names.add(name.replace('NULL + ', '')) |
| else: |
| names.add(name) |
| for co in getattr(func, 'co_consts', tuple()): |
| if co and recurse and iscode(co): |
| names.update(nestedglobals(co, recurse=True)) |
| return list(names) |
| |
| def referredglobals(func, recurse=True, builtin=False): |
| """get the names of objects in the global scope referred to by func""" |
| return globalvars(func, recurse, builtin).keys() |
| |
| def globalvars(func, recurse=True, builtin=False): |
| """get objects defined in global scope that are referred to by func |
| |
| return a dict of {name:object}""" |
| if ismethod(func): func = func.__func__ |
| if isfunction(func): |
| globs = vars(getmodule(sum)).copy() if builtin else {} |
| # get references from within closure |
| orig_func, func = func, set() |
| for obj in orig_func.__closure__ or {}: |
| try: |
| cell_contents = obj.cell_contents |
| except ValueError: # cell is empty |
| pass |
| else: |
| _vars = globalvars(cell_contents, recurse, builtin) or {} |
| func.update(_vars) #XXX: (above) be wary of infinte recursion? |
| globs.update(_vars) |
| # get globals |
| globs.update(orig_func.__globals__ or {}) |
| # get names of references |
| if not recurse: |
| func.update(orig_func.__code__.co_names) |
| else: |
| func.update(nestedglobals(orig_func.__code__)) |
| # find globals for all entries of func |
| for key in func.copy(): #XXX: unnecessary...? |
| nested_func = globs.get(key) |
| if nested_func is orig_func: |
| #func.remove(key) if key in func else None |
| continue #XXX: globalvars(func, False)? |
| func.update(globalvars(nested_func, True, builtin)) |
| elif iscode(func): |
| globs = vars(getmodule(sum)).copy() if builtin else {} |
| #globs.update(globals()) |
| if not recurse: |
| func = func.co_names # get names |
| else: |
| orig_func = func.co_name # to stop infinite recursion |
| func = set(nestedglobals(func)) |
| # find globals for all entries of func |
| for key in func.copy(): #XXX: unnecessary...? |
| if key is orig_func: |
| #func.remove(key) if key in func else None |
| continue #XXX: globalvars(func, False)? |
| nested_func = globs.get(key) |
| func.update(globalvars(nested_func, True, builtin)) |
| else: |
| return {} |
| #NOTE: if name not in __globals__, then we skip it... |
| return dict((name,globs[name]) for name in func if name in globs) |
| |
| |
| def varnames(func): |
| """get names of variables defined by func |
| |
| returns a tuple (local vars, local vars referrenced by nested functions)""" |
| func = code(func) |
| if not iscode(func): |
| return () #XXX: better ((),())? or None? |
| return func.co_varnames, func.co_cellvars |
| |
| |
| def baditems(obj, exact=False, safe=False): #XXX: obj=globals() ? |
| """get items in object that fail to pickle""" |
| if not hasattr(obj,'__iter__'): # is not iterable |
| return [j for j in (badobjects(obj,0,exact,safe),) if j is not None] |
| obj = obj.values() if getattr(obj,'values',None) else obj |
| _obj = [] # can't use a set, as items may be unhashable |
| [_obj.append(badobjects(i,0,exact,safe)) for i in obj if i not in _obj] |
| return [j for j in _obj if j is not None] |
| |
| |
| def badobjects(obj, depth=0, exact=False, safe=False): |
| """get objects that fail to pickle""" |
| from dill import pickles |
| if not depth: |
| if pickles(obj,exact,safe): return None |
| return obj |
| return dict(((attr, badobjects(getattr(obj,attr),depth-1,exact,safe)) \ |
| for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe))) |
| |
| def badtypes(obj, depth=0, exact=False, safe=False): |
| """get types for objects that fail to pickle""" |
| from dill import pickles |
| if not depth: |
| if pickles(obj,exact,safe): return None |
| return type(obj) |
| return dict(((attr, badtypes(getattr(obj,attr),depth-1,exact,safe)) \ |
| for attr in dir(obj) if not pickles(getattr(obj,attr),exact,safe))) |
| |
| def errors(obj, depth=0, exact=False, safe=False): |
| """get errors for objects that fail to pickle""" |
| from dill import pickles, copy |
| if not depth: |
| try: |
| pik = copy(obj) |
| if exact: |
| assert pik == obj, \ |
| "Unpickling produces %s instead of %s" % (pik,obj) |
| assert type(pik) == type(obj), \ |
| "Unpickling produces %s instead of %s" % (type(pik),type(obj)) |
| return None |
| except Exception: |
| import sys |
| return sys.exc_info()[1] |
| _dict = {} |
| for attr in dir(obj): |
| try: |
| _attr = getattr(obj,attr) |
| except Exception: |
| import sys |
| _dict[attr] = sys.exc_info()[1] |
| continue |
| if not pickles(_attr,exact,safe): |
| _dict[attr] = errors(_attr,depth-1,exact,safe) |
| return _dict |
| |
| |
| # EOF |
