| # -*- coding: utf-8 -*- |
| # |
| # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) |
| # Copyright (c) 2008-2015 California Institute of Technology. |
| # Copyright (c) 2016-2023 The Uncertainty Quantification Foundation. |
| # License: 3-clause BSD. The full license text is available at: |
| # - https://github.com/uqfoundation/dill/blob/master/LICENSE |
| """ |
| dill: a utility for serialization of python objects |
| |
| The primary functions in `dill` are :func:`dump` and |
| :func:`dumps` for serialization ("pickling") to a |
| file or to a string, respectively, and :func:`load` |
| and :func:`loads` for deserialization ("unpickling"), |
| similarly, from a file or from a string. Other notable |
| functions are :func:`~dill.dump_module` and |
| :func:`~dill.load_module`, which are used to save and |
| restore module objects, including an intepreter session. |
| |
| Based on code written by Oren Tirosh and Armin Ronacher. |
| Extended to a (near) full set of the builtin types (in types module), |
| and coded to the pickle interface, by <mmckerns@caltech.edu>. |
| Initial port to python3 by Jonathan Dobson, continued by mmckerns. |
| Tested against "all" python types (Std. Lib. CH 1-15 @ 2.7) by mmckerns. |
| Tested against CH16+ Std. Lib. ... TBD. |
| """ |
| |
| from __future__ import annotations |
| |
| __all__ = [ |
| 'dump','dumps','load','loads','copy', |
| 'Pickler','Unpickler','register','pickle','pickles','check', |
| 'DEFAULT_PROTOCOL','HIGHEST_PROTOCOL','HANDLE_FMODE','CONTENTS_FMODE','FILE_FMODE', |
| 'PickleError','PickleWarning','PicklingError','PicklingWarning','UnpicklingError', |
| 'UnpicklingWarning', |
| ] |
| |
| __module__ = 'dill' |
| |
| import warnings |
| from .logger import adapter as logger |
| from .logger import trace as _trace |
| log = logger # backward compatibility (see issue #582) |
| |
| import os |
| import sys |
| diff = None |
| _use_diff = False |
| OLD38 = (sys.hexversion < 0x3080000) |
| OLD39 = (sys.hexversion < 0x3090000) |
| OLD310 = (sys.hexversion < 0x30a0000) |
| OLD312a7 = (sys.hexversion < 0x30c00a7) |
| #XXX: get types from .objtypes ? |
| import builtins as __builtin__ |
| from pickle import _Pickler as StockPickler, Unpickler as StockUnpickler |
| from pickle import GLOBAL, POP |
| from _thread import LockType |
| from _thread import RLock as RLockType |
| #from io import IOBase |
| from types import CodeType, FunctionType, MethodType, GeneratorType, \ |
| TracebackType, FrameType, ModuleType, BuiltinMethodType |
| BufferType = memoryview #XXX: unregistered |
| ClassType = type # no 'old-style' classes |
| EllipsisType = type(Ellipsis) |
| #FileType = IOBase |
| NotImplementedType = type(NotImplemented) |
| SliceType = slice |
| TypeType = type # 'new-style' classes #XXX: unregistered |
| XRangeType = range |
| from types import MappingProxyType as DictProxyType, new_class |
| from pickle import DEFAULT_PROTOCOL, HIGHEST_PROTOCOL, PickleError, PicklingError, UnpicklingError |
| import __main__ as _main_module |
| import marshal |
| import gc |
| # import zlib |
| import abc |
| import dataclasses |
| from weakref import ReferenceType, ProxyType, CallableProxyType |
| from collections import OrderedDict |
| from enum import Enum, EnumMeta |
| from functools import partial |
| from operator import itemgetter, attrgetter |
| GENERATOR_FAIL = False |
| import importlib.machinery |
| EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) |
| try: |
| import ctypes |
| HAS_CTYPES = True |
| # if using `pypy`, pythonapi is not found |
| IS_PYPY = not hasattr(ctypes, 'pythonapi') |
| except ImportError: |
| HAS_CTYPES = False |
| IS_PYPY = False |
| NumpyUfuncType = None |
| NumpyDType = None |
| NumpyArrayType = None |
| try: |
| if not importlib.machinery.PathFinder().find_spec('numpy'): |
| raise ImportError("No module named 'numpy'") |
| NumpyUfuncType = True |
| NumpyDType = True |
| NumpyArrayType = True |
| except ImportError: |
| pass |
| def __hook__(): |
| global NumpyArrayType, NumpyDType, NumpyUfuncType |
| from numpy import ufunc as NumpyUfuncType |
| from numpy import ndarray as NumpyArrayType |
| from numpy import dtype as NumpyDType |
| return True |
| if NumpyArrayType: # then has numpy |
| def ndarraysubclassinstance(obj_type): |
| if all((c.__module__, c.__name__) != ('numpy', 'ndarray') for c in obj_type.__mro__): |
| return False |
| # anything below here is a numpy array (or subclass) instance |
| __hook__() # import numpy (so the following works!!!) |
| # verify that __reduce__ has not been overridden |
| if obj_type.__reduce_ex__ is not NumpyArrayType.__reduce_ex__ \ |
| or obj_type.__reduce__ is not NumpyArrayType.__reduce__: |
| return False |
| return True |
| def numpyufunc(obj_type): |
| return any((c.__module__, c.__name__) == ('numpy', 'ufunc') for c in obj_type.__mro__) |
| def numpydtype(obj_type): |
| if all((c.__module__, c.__name__) != ('numpy', 'dtype') for c in obj_type.__mro__): |
| return False |
| # anything below here is a numpy dtype |
| __hook__() # import numpy (so the following works!!!) |
| return obj_type is type(NumpyDType) # handles subclasses |
| else: |
| def ndarraysubclassinstance(obj): return False |
| def numpyufunc(obj): return False |
| def numpydtype(obj): return False |
| |
| from types import GetSetDescriptorType, ClassMethodDescriptorType, \ |
| WrapperDescriptorType, MethodDescriptorType, MemberDescriptorType, \ |
| MethodWrapperType #XXX: unused |
| |
| # make sure to add these 'hand-built' types to _typemap |
| CellType = type((lambda x: lambda y: x)(0).__closure__[0]) |
| PartialType = type(partial(int, base=2)) |
| SuperType = type(super(Exception, TypeError())) |
| ItemGetterType = type(itemgetter(0)) |
| AttrGetterType = type(attrgetter('__repr__')) |
| |
| try: |
| from functools import _lru_cache_wrapper as LRUCacheType |
| except ImportError: |
| LRUCacheType = None |
| |
| if not isinstance(LRUCacheType, type): |
| LRUCacheType = None |
| |
| def get_file_type(*args, **kwargs): |
| open = kwargs.pop("open", __builtin__.open) |
| f = open(os.devnull, *args, **kwargs) |
| t = type(f) |
| f.close() |
| return t |
| |
| FileType = get_file_type('rb', buffering=0) |
| TextWrapperType = get_file_type('r', buffering=-1) |
| BufferedRandomType = get_file_type('r+b', buffering=-1) |
| BufferedReaderType = get_file_type('rb', buffering=-1) |
| BufferedWriterType = get_file_type('wb', buffering=-1) |
| try: |
| from _pyio import open as _open |
| PyTextWrapperType = get_file_type('r', buffering=-1, open=_open) |
| PyBufferedRandomType = get_file_type('r+b', buffering=-1, open=_open) |
| PyBufferedReaderType = get_file_type('rb', buffering=-1, open=_open) |
| PyBufferedWriterType = get_file_type('wb', buffering=-1, open=_open) |
| except ImportError: |
| PyTextWrapperType = PyBufferedRandomType = PyBufferedReaderType = PyBufferedWriterType = None |
| from io import BytesIO as StringIO |
| InputType = OutputType = None |
| from socket import socket as SocketType |
| #FIXME: additionally calls ForkingPickler.register several times |
| from multiprocessing.reduction import _reduce_socket as reduce_socket |
| try: #pragma: no cover |
| IS_IPYTHON = __IPYTHON__ # is True |
| ExitType = None # IPython.core.autocall.ExitAutocall |
| IPYTHON_SINGLETONS = ('exit', 'quit', 'get_ipython') |
| except NameError: |
| IS_IPYTHON = False |
| try: ExitType = type(exit) # apparently 'exit' can be removed |
| except NameError: ExitType = None |
| IPYTHON_SINGLETONS = () |
| |
| import inspect |
| import typing |
| |
| |
| ### Shims for different versions of Python and dill |
| class Sentinel(object): |
| """ |
| Create a unique sentinel object that is pickled as a constant. |
| """ |
| def __init__(self, name, module_name=None): |
| self.name = name |
| if module_name is None: |
| # Use the calling frame's module |
| self.__module__ = inspect.currentframe().f_back.f_globals['__name__'] |
| else: |
| self.__module__ = module_name # pragma: no cover |
| def __repr__(self): |
| return self.__module__ + '.' + self.name # pragma: no cover |
| def __copy__(self): |
| return self # pragma: no cover |
| def __deepcopy__(self, memo): |
| return self # pragma: no cover |
| def __reduce__(self): |
| return self.name |
| def __reduce_ex__(self, protocol): |
| return self.name |
| |
| from . import _shims |
| from ._shims import Reduce, Getattr |
| |
| ### File modes |
| #: Pickles the file handle, preserving mode. The position of the unpickled |
| #: object is as for a new file handle. |
| HANDLE_FMODE = 0 |
| #: Pickles the file contents, creating a new file if on load the file does |
| #: not exist. The position = min(pickled position, EOF) and mode is chosen |
| #: as such that "best" preserves behavior of the original file. |
| CONTENTS_FMODE = 1 |
| #: Pickles the entire file (handle and contents), preserving mode and position. |
| FILE_FMODE = 2 |
| |
| ### Shorthands (modified from python2.5/lib/pickle.py) |
| def copy(obj, *args, **kwds): |
| """ |
| Use pickling to 'copy' an object (i.e. `loads(dumps(obj))`). |
| |
| See :func:`dumps` and :func:`loads` for keyword arguments. |
| """ |
| ignore = kwds.pop('ignore', Unpickler.settings['ignore']) |
| return loads(dumps(obj, *args, **kwds), ignore=ignore) |
| |
| def dump(obj, file, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None): |
| """ |
| Pickle an object to a file. |
| |
| See :func:`dumps` for keyword arguments. |
| """ |
| from .settings import settings |
| protocol = settings['protocol'] if protocol is None else int(protocol) |
| _kwds = kwds.copy() |
| _kwds.update(dict(byref=byref, fmode=fmode, recurse=recurse)) |
| Pickler(file, protocol, **_kwds).dump(obj) |
| return |
| |
| def dumps(obj, protocol=None, byref=None, fmode=None, recurse=None, **kwds):#, strictio=None): |
| """ |
| Pickle an object to a string. |
| |
| *protocol* is the pickler protocol, as defined for Python *pickle*. |
| |
| If *byref=True*, then dill behaves a lot more like pickle as certain |
| objects (like modules) are pickled by reference as opposed to attempting |
| to pickle the object itself. |
| |
| If *recurse=True*, then objects referred to in the global dictionary |
| are recursively traced and pickled, instead of the default behavior |
| of attempting to store the entire global dictionary. This is needed for |
| functions defined via *exec()*. |
| |
| *fmode* (:const:`HANDLE_FMODE`, :const:`CONTENTS_FMODE`, |
| or :const:`FILE_FMODE`) indicates how file handles will be pickled. |
| For example, when pickling a data file handle for transfer to a remote |
| compute service, *FILE_FMODE* will include the file contents in the |
| pickle and cursor position so that a remote method can operate |
| transparently on an object with an open file handle. |
| |
| Default values for keyword arguments can be set in :mod:`dill.settings`. |
| """ |
| file = StringIO() |
| dump(obj, file, protocol, byref, fmode, recurse, **kwds)#, strictio) |
| return file.getvalue() |
| |
| def load(file, ignore=None, **kwds): |
| """ |
| Unpickle an object from a file. |
| |
| See :func:`loads` for keyword arguments. |
| """ |
| return Unpickler(file, ignore=ignore, **kwds).load() |
| |
| def loads(str, ignore=None, **kwds): |
| """ |
| Unpickle an object from a string. |
| |
| If *ignore=False* then objects whose class is defined in the module |
| *__main__* are updated to reference the existing class in *__main__*, |
| otherwise they are left to refer to the reconstructed type, which may |
| be different. |
| |
| Default values for keyword arguments can be set in :mod:`dill.settings`. |
| """ |
| file = StringIO(str) |
| return load(file, ignore, **kwds) |
| |
| # def dumpzs(obj, protocol=None): |
| # """pickle an object to a compressed string""" |
| # return zlib.compress(dumps(obj, protocol)) |
| |
| # def loadzs(str): |
| # """unpickle an object from a compressed string""" |
| # return loads(zlib.decompress(str)) |
| |
| ### End: Shorthands ### |
| |
| class MetaCatchingDict(dict): |
| def get(self, key, default=None): |
| try: |
| return self[key] |
| except KeyError: |
| return default |
| |
| def __missing__(self, key): |
| if issubclass(key, type): |
| return save_type |
| else: |
| raise KeyError() |
| |
| class PickleWarning(Warning, PickleError): |
| pass |
| |
| class PicklingWarning(PickleWarning, PicklingError): |
| pass |
| |
| class UnpicklingWarning(PickleWarning, UnpicklingError): |
| pass |
| |
| ### Extend the Picklers |
| class Pickler(StockPickler): |
| """python's Pickler extended to interpreter sessions""" |
| dispatch: typing.Dict[type, typing.Callable[[Pickler, typing.Any], None]] \ |
| = MetaCatchingDict(StockPickler.dispatch.copy()) |
| """The dispatch table, a dictionary of serializing functions used |
| by Pickler to save objects of specific types. Use :func:`pickle` |
| or :func:`register` to associate types to custom functions. |
| |
| :meta hide-value: |
| """ |
| _session = False |
| from .settings import settings |
| |
| def __init__(self, file, *args, **kwds): |
| settings = Pickler.settings |
| _byref = kwds.pop('byref', None) |
| #_strictio = kwds.pop('strictio', None) |
| _fmode = kwds.pop('fmode', None) |
| _recurse = kwds.pop('recurse', None) |
| StockPickler.__init__(self, file, *args, **kwds) |
| self._main = _main_module |
| self._diff_cache = {} |
| self._byref = settings['byref'] if _byref is None else _byref |
| self._strictio = False #_strictio |
| self._fmode = settings['fmode'] if _fmode is None else _fmode |
| self._recurse = settings['recurse'] if _recurse is None else _recurse |
| self._postproc = OrderedDict() |
| self._file = file |
| |
| def save(self, obj, save_persistent_id=True): |
| # numpy hack |
| obj_type = type(obj) |
| if NumpyArrayType and not (obj_type is type or obj_type in Pickler.dispatch): |
| # register if the object is a numpy ufunc |
| # thanks to Paul Kienzle for pointing out ufuncs didn't pickle |
| if numpyufunc(obj_type): |
| @register(obj_type) |
| def save_numpy_ufunc(pickler, obj): |
| logger.trace(pickler, "Nu: %s", obj) |
| name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) |
| StockPickler.save_global(pickler, obj, name=name) |
| logger.trace(pickler, "# Nu") |
| return |
| # NOTE: the above 'save' performs like: |
| # import copy_reg |
| # def udump(f): return f.__name__ |
| # def uload(name): return getattr(numpy, name) |
| # copy_reg.pickle(NumpyUfuncType, udump, uload) |
| # register if the object is a numpy dtype |
| if numpydtype(obj_type): |
| @register(obj_type) |
| def save_numpy_dtype(pickler, obj): |
| logger.trace(pickler, "Dt: %s", obj) |
| pickler.save_reduce(_create_dtypemeta, (obj.type,), obj=obj) |
| logger.trace(pickler, "# Dt") |
| return |
| # NOTE: the above 'save' performs like: |
| # import copy_reg |
| # def uload(name): return type(NumpyDType(name)) |
| # def udump(f): return uload, (f.type,) |
| # copy_reg.pickle(NumpyDTypeType, udump, uload) |
| # register if the object is a subclassed numpy array instance |
| if ndarraysubclassinstance(obj_type): |
| @register(obj_type) |
| def save_numpy_array(pickler, obj): |
| logger.trace(pickler, "Nu: (%s, %s)", obj.shape, obj.dtype) |
| npdict = getattr(obj, '__dict__', None) |
| f, args, state = obj.__reduce__() |
| pickler.save_reduce(_create_array, (f,args,state,npdict), obj=obj) |
| logger.trace(pickler, "# Nu") |
| return |
| # end numpy hack |
| |
| if GENERATOR_FAIL and obj_type is GeneratorType: |
| msg = "Can't pickle %s: attribute lookup builtins.generator failed" % GeneratorType |
| raise PicklingError(msg) |
| StockPickler.save(self, obj, save_persistent_id) |
| |
| save.__doc__ = StockPickler.save.__doc__ |
| |
| def dump(self, obj): #NOTE: if settings change, need to update attributes |
| logger.trace_setup(self) |
| StockPickler.dump(self, obj) |
| dump.__doc__ = StockPickler.dump.__doc__ |
| |
| class Unpickler(StockUnpickler): |
| """python's Unpickler extended to interpreter sessions and more types""" |
| from .settings import settings |
| _session = False |
| |
| def find_class(self, module, name): |
| if (module, name) == ('__builtin__', '__main__'): |
| return self._main.__dict__ #XXX: above set w/save_module_dict |
| elif (module, name) == ('__builtin__', 'NoneType'): |
| return type(None) #XXX: special case: NoneType missing |
| if module == 'dill.dill': module = 'dill._dill' |
| return StockUnpickler.find_class(self, module, name) |
| |
| def __init__(self, *args, **kwds): |
| settings = Pickler.settings |
| _ignore = kwds.pop('ignore', None) |
| StockUnpickler.__init__(self, *args, **kwds) |
| self._main = _main_module |
| self._ignore = settings['ignore'] if _ignore is None else _ignore |
| |
| def load(self): #NOTE: if settings change, need to update attributes |
| obj = StockUnpickler.load(self) |
| if type(obj).__module__ == getattr(_main_module, '__name__', '__main__'): |
| if not self._ignore: |
| # point obj class to main |
| try: obj.__class__ = getattr(self._main, type(obj).__name__) |
| except (AttributeError,TypeError): pass # defined in a file |
| #_main_module.__dict__.update(obj.__dict__) #XXX: should update globals ? |
| return obj |
| load.__doc__ = StockUnpickler.load.__doc__ |
| pass |
| |
| ''' |
| def dispatch_table(): |
| """get the dispatch table of registered types""" |
| return Pickler.dispatch |
| ''' |
| |
| pickle_dispatch_copy = StockPickler.dispatch.copy() |
| |
| def pickle(t, func): |
| """expose :attr:`~Pickler.dispatch` table for user-created extensions""" |
| Pickler.dispatch[t] = func |
| return |
| |
| def register(t): |
| """decorator to register types to Pickler's :attr:`~Pickler.dispatch` table""" |
| def proxy(func): |
| Pickler.dispatch[t] = func |
| return func |
| return proxy |
| |
| def _revert_extension(): |
| """drop dill-registered types from pickle's dispatch table""" |
| for type, func in list(StockPickler.dispatch.items()): |
| if func.__module__ == __name__: |
| del StockPickler.dispatch[type] |
| if type in pickle_dispatch_copy: |
| StockPickler.dispatch[type] = pickle_dispatch_copy[type] |
| |
| def use_diff(on=True): |
| """ |
| Reduces size of pickles by only including object which have changed. |
| |
| Decreases pickle size but increases CPU time needed. |
| Also helps avoid some unpickleable objects. |
| MUST be called at start of script, otherwise changes will not be recorded. |
| """ |
| global _use_diff, diff |
| _use_diff = on |
| if _use_diff and diff is None: |
| try: |
| from . import diff as d |
| except ImportError: |
| import diff as d |
| diff = d |
| |
| def _create_typemap(): |
| import types |
| d = dict(list(__builtin__.__dict__.items()) + \ |
| list(types.__dict__.items())).items() |
| for key, value in d: |
| if getattr(value, '__module__', None) == 'builtins' \ |
| and type(value) is type: |
| yield key, value |
| return |
| _reverse_typemap = dict(_create_typemap()) |
| _reverse_typemap.update({ |
| 'PartialType': PartialType, |
| 'SuperType': SuperType, |
| 'ItemGetterType': ItemGetterType, |
| 'AttrGetterType': AttrGetterType, |
| }) |
| if sys.hexversion < 0x30800a2: |
| _reverse_typemap.update({ |
| 'CellType': CellType, |
| }) |
| |
| # "Incidental" implementation specific types. Unpickling these types in another |
| # implementation of Python (PyPy -> CPython) is not guaranteed to work |
| |
| # This dictionary should contain all types that appear in Python implementations |
| # but are not defined in https://docs.python.org/3/library/types.html#standard-interpreter-types |
| x=OrderedDict() |
| _incedental_reverse_typemap = { |
| 'FileType': FileType, |
| 'BufferedRandomType': BufferedRandomType, |
| 'BufferedReaderType': BufferedReaderType, |
| 'BufferedWriterType': BufferedWriterType, |
| 'TextWrapperType': TextWrapperType, |
| 'PyBufferedRandomType': PyBufferedRandomType, |
| 'PyBufferedReaderType': PyBufferedReaderType, |
| 'PyBufferedWriterType': PyBufferedWriterType, |
| 'PyTextWrapperType': PyTextWrapperType, |
| } |
| |
| _incedental_reverse_typemap.update({ |
| "DictKeysType": type({}.keys()), |
| "DictValuesType": type({}.values()), |
| "DictItemsType": type({}.items()), |
| |
| "OdictKeysType": type(x.keys()), |
| "OdictValuesType": type(x.values()), |
| "OdictItemsType": type(x.items()), |
| }) |
| |
| if ExitType: |
| _incedental_reverse_typemap['ExitType'] = ExitType |
| if InputType: |
| _incedental_reverse_typemap['InputType'] = InputType |
| _incedental_reverse_typemap['OutputType'] = OutputType |
| |
| ''' |
| try: |
| import symtable |
| _incedental_reverse_typemap["SymtableEntryType"] = type(symtable.symtable("", "string", "exec")._table) |
| except: #FIXME: fails to pickle |
| pass |
| |
| if sys.hexversion >= 0x30a00a0: |
| _incedental_reverse_typemap['LineIteratorType'] = type(compile('3', '', 'eval').co_lines()) |
| ''' |
| |
| if sys.hexversion >= 0x30b00b0: |
| from types import GenericAlias |
| _incedental_reverse_typemap["GenericAliasIteratorType"] = type(iter(GenericAlias(list, (int,)))) |
| ''' |
| _incedental_reverse_typemap['PositionsIteratorType'] = type(compile('3', '', 'eval').co_positions()) |
| ''' |
| |
| try: |
| import winreg |
| _incedental_reverse_typemap["HKEYType"] = winreg.HKEYType |
| except ImportError: |
| pass |
| |
| _reverse_typemap.update(_incedental_reverse_typemap) |
| _incedental_types = set(_incedental_reverse_typemap.values()) |
| |
| del x |
| |
| _typemap = dict((v, k) for k, v in _reverse_typemap.items()) |
| |
| def _unmarshal(string): |
| return marshal.loads(string) |
| |
| def _load_type(name): |
| return _reverse_typemap[name] |
| |
| def _create_type(typeobj, *args): |
| return typeobj(*args) |
| |
| def _create_function(fcode, fglobals, fname=None, fdefaults=None, |
| fclosure=None, fdict=None, fkwdefaults=None): |
| # same as FunctionType, but enable passing __dict__ to new function, |
| # __dict__ is the storehouse for attributes added after function creation |
| func = FunctionType(fcode, fglobals or dict(), fname, fdefaults, fclosure) |
| if fdict is not None: |
| func.__dict__.update(fdict) #XXX: better copy? option to copy? |
| if fkwdefaults is not None: |
| func.__kwdefaults__ = fkwdefaults |
| # 'recurse' only stores referenced modules/objects in fglobals, |
| # thus we need to make sure that we have __builtins__ as well |
| if "__builtins__" not in func.__globals__: |
| func.__globals__["__builtins__"] = globals()["__builtins__"] |
| # assert id(fglobals) == id(func.__globals__) |
| return func |
| |
| class match: |
| """ |
| Make avaialable a limited structural pattern matching-like syntax for Python < 3.10 |
| |
| Patterns can be only tuples (without types) currently. |
| Inspired by the package pattern-matching-PEP634. |
| |
| Usage: |
| >>> with match(args) as m: |
| >>> if m.case(('x', 'y')): |
| >>> # use m.x and m.y |
| >>> elif m.case(('x', 'y', 'z')): |
| >>> # use m.x, m.y and m.z |
| |
| Equivalent native code for Python >= 3.10: |
| >>> match args: |
| >>> case (x, y): |
| >>> # use x and y |
| >>> case (x, y, z): |
| >>> # use x, y and z |
| """ |
| def __init__(self, value): |
| self.value = value |
| self._fields = None |
| def __enter__(self): |
| return self |
| def __exit__(self, *exc_info): |
| return False |
| def case(self, args): # *args, **kwargs): |
| """just handles tuple patterns""" |
| if len(self.value) != len(args): # + len(kwargs): |
| return False |
| #if not all(isinstance(arg, pat) for arg, pat in zip(self.value[len(args):], kwargs.values())): |
| # return False |
| self.args = args # (*args, *kwargs) |
| return True |
| @property |
| def fields(self): |
| # Only bind names to values if necessary. |
| if self._fields is None: |
| self._fields = dict(zip(self.args, self.value)) |
| return self._fields |
| def __getattr__(self, item): |
| return self.fields[item] |
| |
| ALL_CODE_PARAMS = [ |
| # Version New attribute CodeType parameters |
| ((3,11,'a'), 'co_endlinetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable endlinetable columntable exceptiontable freevars cellvars'), |
| ((3,11), 'co_exceptiontable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name qualname firstlineno linetable exceptiontable freevars cellvars'), |
| ((3,10), 'co_linetable', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno linetable freevars cellvars'), |
| ((3,8), 'co_posonlyargcount', 'argcount posonlyargcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), |
| ((3,7), 'co_kwonlyargcount', 'argcount kwonlyargcount nlocals stacksize flags code consts names varnames filename name firstlineno lnotab freevars cellvars'), |
| ] |
| for version, new_attr, params in ALL_CODE_PARAMS: |
| if hasattr(CodeType, new_attr): |
| CODE_VERSION = version |
| CODE_PARAMS = params.split() |
| break |
| ENCODE_PARAMS = set(CODE_PARAMS).intersection( |
| ['code', 'lnotab', 'linetable', 'endlinetable', 'columntable', 'exceptiontable']) |
| |
| def _create_code(*args): |
| if not isinstance(args[0], int): # co_lnotab stored from >= 3.10 |
| LNOTAB, *args = args |
| else: # from < 3.10 (or pre-LNOTAB storage) |
| LNOTAB = b'' |
| |
| with match(args) as m: |
| # Python 3.11/3.12a (18 members) |
| if m.case(( |
| 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] |
| 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] |
| 'linetable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] |
| )): |
| if CODE_VERSION == (3,11): |
| return CodeType( |
| *args[:6], |
| args[6].encode() if hasattr(args[6], 'encode') else args[6], # code |
| *args[7:14], |
| args[14].encode() if hasattr(args[14], 'encode') else args[14], # linetable |
| args[15].encode() if hasattr(args[15], 'encode') else args[15], # exceptiontable |
| args[16], |
| args[17], |
| ) |
| fields = m.fields |
| # Python 3.10 or 3.8/3.9 (16 members) |
| elif m.case(( |
| 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] |
| 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[6:13] |
| 'LNOTAB_OR_LINETABLE', 'freevars', 'cellvars' # args[13:] |
| )): |
| if CODE_VERSION == (3,10) or CODE_VERSION == (3,8): |
| return CodeType( |
| *args[:6], |
| args[6].encode() if hasattr(args[6], 'encode') else args[6], # code |
| *args[7:13], |
| args[13].encode() if hasattr(args[13], 'encode') else args[13], # lnotab/linetable |
| args[14], |
| args[15], |
| ) |
| fields = m.fields |
| if CODE_VERSION >= (3,10): |
| fields['linetable'] = m.LNOTAB_OR_LINETABLE |
| else: |
| fields['lnotab'] = LNOTAB if LNOTAB else m.LNOTAB_OR_LINETABLE |
| # Python 3.7 (15 args) |
| elif m.case(( |
| 'argcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:5] |
| 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'firstlineno', # args[5:12] |
| 'lnotab', 'freevars', 'cellvars' # args[12:] |
| )): |
| if CODE_VERSION == (3,7): |
| return CodeType( |
| *args[:5], |
| args[5].encode() if hasattr(args[5], 'encode') else args[5], # code |
| *args[6:12], |
| args[12].encode() if hasattr(args[12], 'encode') else args[12], # lnotab |
| args[13], |
| args[14], |
| ) |
| fields = m.fields |
| # Python 3.11a (20 members) |
| elif m.case(( |
| 'argcount', 'posonlyargcount', 'kwonlyargcount', 'nlocals', 'stacksize', 'flags', # args[0:6] |
| 'code', 'consts', 'names', 'varnames', 'filename', 'name', 'qualname', 'firstlineno', # args[6:14] |
| 'linetable', 'endlinetable', 'columntable', 'exceptiontable', 'freevars', 'cellvars' # args[14:] |
| )): |
| if CODE_VERSION == (3,11,'a'): |
| return CodeType( |
| *args[:6], |
| args[6].encode() if hasattr(args[6], 'encode') else args[6], # code |
| *args[7:14], |
| *(a.encode() if hasattr(a, 'encode') else a for a in args[14:18]), # linetable-exceptiontable |
| args[18], |
| args[19], |
| ) |
| fields = m.fields |
| else: |
| raise UnpicklingError("pattern match for code object failed") |
| |
| # The args format doesn't match this version. |
| fields.setdefault('posonlyargcount', 0) # from python <= 3.7 |
| fields.setdefault('lnotab', LNOTAB) # from python >= 3.10 |
| fields.setdefault('linetable', b'') # from python <= 3.9 |
| fields.setdefault('qualname', fields['name']) # from python <= 3.10 |
| fields.setdefault('exceptiontable', b'') # from python <= 3.10 |
| fields.setdefault('endlinetable', None) # from python != 3.11a |
| fields.setdefault('columntable', None) # from python != 3.11a |
| |
| args = (fields[k].encode() if k in ENCODE_PARAMS and hasattr(fields[k], 'encode') else fields[k] |
| for k in CODE_PARAMS) |
| return CodeType(*args) |
| |
| def _create_ftype(ftypeobj, func, args, kwds): |
| if kwds is None: |
| kwds = {} |
| if args is None: |
| args = () |
| return ftypeobj(func, *args, **kwds) |
| |
| def _create_typing_tuple(argz, *args): #NOTE: workaround python/cpython#94245 |
| if not argz: |
| return typing.Tuple[()].copy_with(()) |
| if argz == ((),): |
| return typing.Tuple[()] |
| return typing.Tuple[argz] |
| |
| def _create_lock(locked, *args): #XXX: ignores 'blocking' |
| from threading import Lock |
| lock = Lock() |
| if locked: |
| if not lock.acquire(False): |
| raise UnpicklingError("Cannot acquire lock") |
| return lock |
| |
| def _create_rlock(count, owner, *args): #XXX: ignores 'blocking' |
| lock = RLockType() |
| if owner is not None: |
| lock._acquire_restore((count, owner)) |
| if owner and not lock._is_owned(): |
| raise UnpicklingError("Cannot acquire lock") |
| return lock |
| |
| # thanks to matsjoyce for adding all the different file modes |
| def _create_filehandle(name, mode, position, closed, open, strictio, fmode, fdata): # buffering=0 |
| # only pickles the handle, not the file contents... good? or StringIO(data)? |
| # (for file contents see: http://effbot.org/librarybook/copy-reg.htm) |
| # NOTE: handle special cases first (are there more special cases?) |
| names = {'<stdin>':sys.__stdin__, '<stdout>':sys.__stdout__, |
| '<stderr>':sys.__stderr__} #XXX: better fileno=(0,1,2) ? |
| if name in list(names.keys()): |
| f = names[name] #XXX: safer "f=sys.stdin" |
| elif name == '<tmpfile>': |
| f = os.tmpfile() |
| elif name == '<fdopen>': |
| import tempfile |
| f = tempfile.TemporaryFile(mode) |
| else: |
| try: |
| exists = os.path.exists(name) |
| except Exception: |
| exists = False |
| if not exists: |
| if strictio: |
| raise FileNotFoundError("[Errno 2] No such file or directory: '%s'" % name) |
| elif "r" in mode and fmode != FILE_FMODE: |
| name = '<fdopen>' # or os.devnull? |
| current_size = 0 # or maintain position? |
| else: |
| current_size = os.path.getsize(name) |
| |
| if position > current_size: |
| if strictio: |
| raise ValueError("invalid buffer size") |
| elif fmode == CONTENTS_FMODE: |
| position = current_size |
| # try to open the file by name |
| # NOTE: has different fileno |
| try: |
| #FIXME: missing: *buffering*, encoding, softspace |
| if fmode == FILE_FMODE: |
| f = open(name, mode if "w" in mode else "w") |
| f.write(fdata) |
| if "w" not in mode: |
| f.close() |
| f = open(name, mode) |
| elif name == '<fdopen>': # file did not exist |
| import tempfile |
| f = tempfile.TemporaryFile(mode) |
| # treat x mode as w mode |
| elif fmode == CONTENTS_FMODE \ |
| and ("w" in mode or "x" in mode): |
| # stop truncation when opening |
| flags = os.O_CREAT |
| if "+" in mode: |
| flags |= os.O_RDWR |
| else: |
| flags |= os.O_WRONLY |
| f = os.fdopen(os.open(name, flags), mode) |
| # set name to the correct value |
| r = getattr(f, "buffer", f) |
| r = getattr(r, "raw", r) |
| r.name = name |
| assert f.name == name |
| else: |
| f = open(name, mode) |
| except (IOError, FileNotFoundError): |
| err = sys.exc_info()[1] |
| raise UnpicklingError(err) |
| if closed: |
| f.close() |
| elif position >= 0 and fmode != HANDLE_FMODE: |
| f.seek(position) |
| return f |
| |
| def _create_stringi(value, position, closed): |
| f = StringIO(value) |
| if closed: f.close() |
| else: f.seek(position) |
| return f |
| |
| def _create_stringo(value, position, closed): |
| f = StringIO() |
| if closed: f.close() |
| else: |
| f.write(value) |
| f.seek(position) |
| return f |
| |
| class _itemgetter_helper(object): |
| def __init__(self): |
| self.items = [] |
| def __getitem__(self, item): |
| self.items.append(item) |
| return |
| |
| class _attrgetter_helper(object): |
| def __init__(self, attrs, index=None): |
| self.attrs = attrs |
| self.index = index |
| def __getattribute__(self, attr): |
| attrs = object.__getattribute__(self, "attrs") |
| index = object.__getattribute__(self, "index") |
| if index is None: |
| index = len(attrs) |
| attrs.append(attr) |
| else: |
| attrs[index] = ".".join([attrs[index], attr]) |
| return type(self)(attrs, index) |
| |
| class _dictproxy_helper(dict): |
| def __ror__(self, a): |
| return a |
| |
| _dictproxy_helper_instance = _dictproxy_helper() |
| |
| __d = {} |
| try: |
| # In CPython 3.9 and later, this trick can be used to exploit the |
| # implementation of the __or__ function of MappingProxyType to get the true |
| # mapping referenced by the proxy. It may work for other implementations, |
| # but is not guaranteed. |
| MAPPING_PROXY_TRICK = __d is (DictProxyType(__d) | _dictproxy_helper_instance) |
| except Exception: |
| MAPPING_PROXY_TRICK = False |
| del __d |
| |
| # _CELL_REF and _CELL_EMPTY are used to stay compatible with versions of dill |
| # whose _create_cell functions do not have a default value. |
| # _CELL_REF can be safely removed entirely (replaced by empty tuples for calls |
| # to _create_cell) once breaking changes are allowed. |
| _CELL_REF = None |
| _CELL_EMPTY = Sentinel('_CELL_EMPTY') |
| |
| def _create_cell(contents=None): |
| if contents is not _CELL_EMPTY: |
| value = contents |
| return (lambda: value).__closure__[0] |
| |
| def _create_weakref(obj, *args): |
| from weakref import ref |
| if obj is None: # it's dead |
| from collections import UserDict |
| return ref(UserDict(), *args) |
| return ref(obj, *args) |
| |
| def _create_weakproxy(obj, callable=False, *args): |
| from weakref import proxy |
| if obj is None: # it's dead |
| if callable: return proxy(lambda x:x, *args) |
| from collections import UserDict |
| return proxy(UserDict(), *args) |
| return proxy(obj, *args) |
| |
| def _eval_repr(repr_str): |
| return eval(repr_str) |
| |
| def _create_array(f, args, state, npdict=None): |
| #array = numpy.core.multiarray._reconstruct(*args) |
| array = f(*args) |
| array.__setstate__(state) |
| if npdict is not None: # we also have saved state in __dict__ |
| array.__dict__.update(npdict) |
| return array |
| |
| def _create_dtypemeta(scalar_type): |
| if NumpyDType is True: __hook__() # a bit hacky I think |
| if scalar_type is None: |
| return NumpyDType |
| return type(NumpyDType(scalar_type)) |
| |
| def _create_namedtuple(name, fieldnames, modulename, defaults=None): |
| class_ = _import_module(modulename + '.' + name, safe=True) |
| if class_ is not None: |
| return class_ |
| import collections |
| t = collections.namedtuple(name, fieldnames, defaults=defaults, module=modulename) |
| return t |
| |
| def _create_capsule(pointer, name, context, destructor): |
| attr_found = False |
| try: |
| # based on https://github.com/python/cpython/blob/f4095e53ab708d95e019c909d5928502775ba68f/Objects/capsule.c#L209-L231 |
| uname = name.decode('utf8') |
| for i in range(1, uname.count('.')+1): |
| names = uname.rsplit('.', i) |
| try: |
| module = __import__(names[0]) |
| except ImportError: |
| pass |
| obj = module |
| for attr in names[1:]: |
| obj = getattr(obj, attr) |
| capsule = obj |
| attr_found = True |
| break |
| except Exception: |
| pass |
| |
| if attr_found: |
| if _PyCapsule_IsValid(capsule, name): |
| return capsule |
| raise UnpicklingError("%s object exists at %s but a PyCapsule object was expected." % (type(capsule), name)) |
| else: |
| #warnings.warn('Creating a new PyCapsule %s for a C data structure that may not be present in memory. Segmentation faults or other memory errors are possible.' % (name,), UnpicklingWarning) |
| capsule = _PyCapsule_New(pointer, name, destructor) |
| _PyCapsule_SetContext(capsule, context) |
| return capsule |
| |
| def _getattr(objclass, name, repr_str): |
| # hack to grab the reference directly |
| try: #XXX: works only for __builtin__ ? |
| attr = repr_str.split("'")[3] |
| return eval(attr+'.__dict__["'+name+'"]') |
| except Exception: |
| try: |
| attr = objclass.__dict__ |
| if type(attr) is DictProxyType: |
| attr = attr[name] |
| else: |
| attr = getattr(objclass,name) |
| except (AttributeError, KeyError): |
| attr = getattr(objclass,name) |
| return attr |
| |
| def _get_attr(self, name): |
| # stop recursive pickling |
| return getattr(self, name, None) or getattr(__builtin__, name) |
| |
| def _import_module(import_name, safe=False): |
| try: |
| if import_name.startswith('__runtime__.'): |
| return sys.modules[import_name] |
| elif '.' in import_name: |
| items = import_name.split('.') |
| module = '.'.join(items[:-1]) |
| obj = items[-1] |
| else: |
| return __import__(import_name) |
| return getattr(__import__(module, None, None, [obj]), obj) |
| except (ImportError, AttributeError, KeyError): |
| if safe: |
| return None |
| raise |
| |
| # https://github.com/python/cpython/blob/a8912a0f8d9eba6d502c37d522221f9933e976db/Lib/pickle.py#L322-L333 |
| def _getattribute(obj, name): |
| for subpath in name.split('.'): |
| if subpath == '<locals>': |
| raise AttributeError("Can't get local attribute {!r} on {!r}" |
| .format(name, obj)) |
| try: |
| parent = obj |
| obj = getattr(obj, subpath) |
| except AttributeError: |
| raise AttributeError("Can't get attribute {!r} on {!r}" |
| .format(name, obj)) |
| return obj, parent |
| |
| def _locate_function(obj, pickler=None): |
| module_name = getattr(obj, '__module__', None) |
| if module_name in ['__main__', None] or \ |
| pickler and is_dill(pickler, child=False) and pickler._session and module_name == pickler._main.__name__: |
| return False |
| if hasattr(obj, '__qualname__'): |
| module = _import_module(module_name, safe=True) |
| try: |
| found, _ = _getattribute(module, obj.__qualname__) |
| return found is obj |
| except AttributeError: |
| return False |
| else: |
| found = _import_module(module_name + '.' + obj.__name__, safe=True) |
| return found is obj |
| |
| |
| def _setitems(dest, source): |
| for k, v in source.items(): |
| dest[k] = v |
| |
| |
| def _save_with_postproc(pickler, reduction, is_pickler_dill=None, obj=Getattr.NO_DEFAULT, postproc_list=None): |
| if obj is Getattr.NO_DEFAULT: |
| obj = Reduce(reduction) # pragma: no cover |
| |
| if is_pickler_dill is None: |
| is_pickler_dill = is_dill(pickler, child=True) |
| if is_pickler_dill: |
| # assert id(obj) not in pickler._postproc, str(obj) + ' already pushed on stack!' |
| # if not hasattr(pickler, 'x'): pickler.x = 0 |
| # print(pickler.x*' ', 'push', obj, id(obj), pickler._recurse) |
| # pickler.x += 1 |
| if postproc_list is None: |
| postproc_list = [] |
| |
| # Recursive object not supported. Default to a global instead. |
| if id(obj) in pickler._postproc: |
| name = '%s.%s ' % (obj.__module__, getattr(obj, '__qualname__', obj.__name__)) if hasattr(obj, '__module__') else '' |
| warnings.warn('Cannot pickle %r: %shas recursive self-references that trigger a RecursionError.' % (obj, name), PicklingWarning) |
| pickler.save_global(obj) |
| return |
| pickler._postproc[id(obj)] = postproc_list |
| |
| # TODO: Use state_setter in Python 3.8 to allow for faster cPickle implementations |
| pickler.save_reduce(*reduction, obj=obj) |
| |
| if is_pickler_dill: |
| # pickler.x -= 1 |
| # print(pickler.x*' ', 'pop', obj, id(obj)) |
| postproc = pickler._postproc.pop(id(obj)) |
| # assert postproc_list == postproc, 'Stack tampered!' |
| for reduction in reversed(postproc): |
| if reduction[0] is _setitems: |
| # use the internal machinery of pickle.py to speedup when |
| # updating a dictionary in postproc |
| dest, source = reduction[1] |
| if source: |
| pickler.write(pickler.get(pickler.memo[id(dest)][0])) |
| pickler._batch_setitems(iter(source.items())) |
| else: |
| # Updating with an empty dictionary. Same as doing nothing. |
| continue |
| else: |
| pickler.save_reduce(*reduction) |
| # pop None created by calling preprocessing step off stack |
| pickler.write(POP) |
| |
| #@register(CodeType) |
| #def save_code(pickler, obj): |
| # logger.trace(pickler, "Co: %s", obj) |
| # pickler.save_reduce(_unmarshal, (marshal.dumps(obj),), obj=obj) |
| # logger.trace(pickler, "# Co") |
| # return |
| |
| # The following function is based on 'save_codeobject' from 'cloudpickle' |
| # Copyright (c) 2012, Regents of the University of California. |
| # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_. |
| # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE |
| @register(CodeType) |
| def save_code(pickler, obj): |
| logger.trace(pickler, "Co: %s", obj) |
| if hasattr(obj, "co_endlinetable"): # python 3.11a (20 args) |
| args = ( |
| obj.co_lnotab, # for < python 3.10 [not counted in args] |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, |
| obj.co_firstlineno, obj.co_linetable, obj.co_endlinetable, |
| obj.co_columntable, obj.co_exceptiontable, obj.co_freevars, |
| obj.co_cellvars |
| ) |
| elif hasattr(obj, "co_exceptiontable"): # python 3.11 (18 args) |
| with warnings.catch_warnings(): |
| if not OLD312a7: # issue 597 |
| warnings.filterwarnings('ignore', category=DeprecationWarning) |
| args = ( |
| obj.co_lnotab, # for < python 3.10 [not counted in args] |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, obj.co_qualname, |
| obj.co_firstlineno, obj.co_linetable, obj.co_exceptiontable, |
| obj.co_freevars, obj.co_cellvars |
| ) |
| elif hasattr(obj, "co_linetable"): # python 3.10 (16 args) |
| args = ( |
| obj.co_lnotab, # for < python 3.10 [not counted in args] |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, |
| obj.co_firstlineno, obj.co_linetable, obj.co_freevars, |
| obj.co_cellvars |
| ) |
| elif hasattr(obj, "co_posonlyargcount"): # python 3.8 (16 args) |
| args = ( |
| obj.co_argcount, obj.co_posonlyargcount, |
| obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, |
| obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, |
| obj.co_varnames, obj.co_filename, obj.co_name, |
| obj.co_firstlineno, obj.co_lnotab, obj.co_freevars, |
| obj.co_cellvars |
| ) |
| else: # python 3.7 (15 args) |
| args = ( |
| obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, |
| obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, |
| obj.co_names, obj.co_varnames, obj.co_filename, |
| obj.co_name, obj.co_firstlineno, obj.co_lnotab, |
| obj.co_freevars, obj.co_cellvars |
| ) |
| |
| pickler.save_reduce(_create_code, args, obj=obj) |
| logger.trace(pickler, "# Co") |
| return |
| |
| def _repr_dict(obj): |
| """Make a short string representation of a dictionary.""" |
| return "<%s object at %#012x>" % (type(obj).__name__, id(obj)) |
| |
| @register(dict) |
| def save_module_dict(pickler, obj): |
| if is_dill(pickler, child=False) and obj == pickler._main.__dict__ and \ |
| not (pickler._session and pickler._first_pass): |
| logger.trace(pickler, "D1: %s", _repr_dict(obj)) # obj |
| pickler.write(bytes('c__builtin__\n__main__\n', 'UTF-8')) |
| logger.trace(pickler, "# D1") |
| elif (not is_dill(pickler, child=False)) and (obj == _main_module.__dict__): |
| logger.trace(pickler, "D3: %s", _repr_dict(obj)) # obj |
| pickler.write(bytes('c__main__\n__dict__\n', 'UTF-8')) #XXX: works in general? |
| logger.trace(pickler, "# D3") |
| elif '__name__' in obj and obj != _main_module.__dict__ \ |
| and type(obj['__name__']) is str \ |
| and obj is getattr(_import_module(obj['__name__'],True), '__dict__', None): |
| logger.trace(pickler, "D4: %s", _repr_dict(obj)) # obj |
| pickler.write(bytes('c%s\n__dict__\n' % obj['__name__'], 'UTF-8')) |
| logger.trace(pickler, "# D4") |
| else: |
| logger.trace(pickler, "D2: %s", _repr_dict(obj)) # obj |
| if is_dill(pickler, child=False) and pickler._session: |
| # we only care about session the first pass thru |
| pickler._first_pass = False |
| StockPickler.save_dict(pickler, obj) |
| logger.trace(pickler, "# D2") |
| return |
| |
| |
| if not OLD310 and MAPPING_PROXY_TRICK: |
| def save_dict_view(dicttype): |
| def save_dict_view_for_function(func): |
| def _save_dict_view(pickler, obj): |
| logger.trace(pickler, "Dkvi: <%s>", obj) |
| mapping = obj.mapping | _dictproxy_helper_instance |
| pickler.save_reduce(func, (mapping,), obj=obj) |
| logger.trace(pickler, "# Dkvi") |
| return _save_dict_view |
| return [ |
| (funcname, save_dict_view_for_function(getattr(dicttype, funcname))) |
| for funcname in ('keys', 'values', 'items') |
| ] |
| else: |
| # The following functions are based on 'cloudpickle' |
| # https://github.com/cloudpipe/cloudpickle/blob/5d89947288a18029672596a4d719093cc6d5a412/cloudpickle/cloudpickle.py#L922-L940 |
| # Copyright (c) 2012, Regents of the University of California. |
| # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_. |
| # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE |
| def save_dict_view(dicttype): |
| def save_dict_keys(pickler, obj): |
| logger.trace(pickler, "Dk: <%s>", obj) |
| dict_constructor = _shims.Reduce(dicttype.fromkeys, (list(obj),)) |
| pickler.save_reduce(dicttype.keys, (dict_constructor,), obj=obj) |
| logger.trace(pickler, "# Dk") |
| |
| def save_dict_values(pickler, obj): |
| logger.trace(pickler, "Dv: <%s>", obj) |
| dict_constructor = _shims.Reduce(dicttype, (enumerate(obj),)) |
| pickler.save_reduce(dicttype.values, (dict_constructor,), obj=obj) |
| logger.trace(pickler, "# Dv") |
| |
| def save_dict_items(pickler, obj): |
| logger.trace(pickler, "Di: <%s>", obj) |
| pickler.save_reduce(dicttype.items, (dicttype(obj),), obj=obj) |
| logger.trace(pickler, "# Di") |
| |
| return ( |
| ('keys', save_dict_keys), |
| ('values', save_dict_values), |
| ('items', save_dict_items) |
| ) |
| |
| for __dicttype in ( |
| dict, |
| OrderedDict |
| ): |
| __obj = __dicttype() |
| for __funcname, __savefunc in save_dict_view(__dicttype): |
| __tview = type(getattr(__obj, __funcname)()) |
| if __tview not in Pickler.dispatch: |
| Pickler.dispatch[__tview] = __savefunc |
| del __dicttype, __obj, __funcname, __tview, __savefunc |
| |
| |
| @register(ClassType) |
| def save_classobj(pickler, obj): #FIXME: enable pickler._byref |
| if not _locate_function(obj, pickler): |
| logger.trace(pickler, "C1: %s", obj) |
| pickler.save_reduce(ClassType, (obj.__name__, obj.__bases__, |
| obj.__dict__), obj=obj) |
| #XXX: or obj.__dict__.copy()), obj=obj) ? |
| logger.trace(pickler, "# C1") |
| else: |
| logger.trace(pickler, "C2: %s", obj) |
| name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) |
| StockPickler.save_global(pickler, obj, name=name) |
| logger.trace(pickler, "# C2") |
| return |
| |
| @register(typing._GenericAlias) |
| def save_generic_alias(pickler, obj): |
| args = obj.__args__ |
| if type(obj.__reduce__()) is str: |
| logger.trace(pickler, "Ga0: %s", obj) |
| StockPickler.save_global(pickler, obj, name=obj.__reduce__()) |
| logger.trace(pickler, "# Ga0") |
| elif obj.__origin__ is tuple and (not args or args == ((),)): |
| logger.trace(pickler, "Ga1: %s", obj) |
| pickler.save_reduce(_create_typing_tuple, (args,), obj=obj) |
| logger.trace(pickler, "# Ga1") |
| else: |
| logger.trace(pickler, "Ga2: %s", obj) |
| StockPickler.save_reduce(pickler, *obj.__reduce__(), obj=obj) |
| logger.trace(pickler, "# Ga2") |
| return |
| |
| @register(LockType) |
| def save_lock(pickler, obj): |
| logger.trace(pickler, "Lo: %s", obj) |
| pickler.save_reduce(_create_lock, (obj.locked(),), obj=obj) |
| logger.trace(pickler, "# Lo") |
| return |
| |
| @register(RLockType) |
| def save_rlock(pickler, obj): |
| logger.trace(pickler, "RL: %s", obj) |
| r = obj.__repr__() # don't use _release_save as it unlocks the lock |
| count = int(r.split('count=')[1].split()[0].rstrip('>')) |
| owner = int(r.split('owner=')[1].split()[0]) |
| pickler.save_reduce(_create_rlock, (count,owner,), obj=obj) |
| logger.trace(pickler, "# RL") |
| return |
| |
| #@register(SocketType) #FIXME: causes multiprocess test_pickling FAIL |
| def save_socket(pickler, obj): |
| logger.trace(pickler, "So: %s", obj) |
| pickler.save_reduce(*reduce_socket(obj)) |
| logger.trace(pickler, "# So") |
| return |
| |
| def _save_file(pickler, obj, open_): |
| if obj.closed: |
| position = 0 |
| else: |
| obj.flush() |
| if obj in (sys.__stdout__, sys.__stderr__, sys.__stdin__): |
| position = -1 |
| else: |
| position = obj.tell() |
| if is_dill(pickler, child=True) and pickler._fmode == FILE_FMODE: |
| f = open_(obj.name, "r") |
| fdata = f.read() |
| f.close() |
| else: |
| fdata = "" |
| if is_dill(pickler, child=True): |
| strictio = pickler._strictio |
| fmode = pickler._fmode |
| else: |
| strictio = False |
| fmode = 0 # HANDLE_FMODE |
| pickler.save_reduce(_create_filehandle, (obj.name, obj.mode, position, |
| obj.closed, open_, strictio, |
| fmode, fdata), obj=obj) |
| return |
| |
| |
| @register(FileType) #XXX: in 3.x has buffer=0, needs different _create? |
| @register(BufferedRandomType) |
| @register(BufferedReaderType) |
| @register(BufferedWriterType) |
| @register(TextWrapperType) |
| def save_file(pickler, obj): |
| logger.trace(pickler, "Fi: %s", obj) |
| f = _save_file(pickler, obj, open) |
| logger.trace(pickler, "# Fi") |
| return f |
| |
| if PyTextWrapperType: |
| @register(PyBufferedRandomType) |
| @register(PyBufferedReaderType) |
| @register(PyBufferedWriterType) |
| @register(PyTextWrapperType) |
| def save_file(pickler, obj): |
| logger.trace(pickler, "Fi: %s", obj) |
| f = _save_file(pickler, obj, _open) |
| logger.trace(pickler, "# Fi") |
| return f |
| |
| # The following two functions are based on 'saveCStringIoInput' |
| # and 'saveCStringIoOutput' from spickle |
| # Copyright (c) 2011 by science+computing ag |
| # License: http://www.apache.org/licenses/LICENSE-2.0 |
| if InputType: |
| @register(InputType) |
| def save_stringi(pickler, obj): |
| logger.trace(pickler, "Io: %s", obj) |
| if obj.closed: |
| value = ''; position = 0 |
| else: |
| value = obj.getvalue(); position = obj.tell() |
| pickler.save_reduce(_create_stringi, (value, position, \ |
| obj.closed), obj=obj) |
| logger.trace(pickler, "# Io") |
| return |
| |
| @register(OutputType) |
| def save_stringo(pickler, obj): |
| logger.trace(pickler, "Io: %s", obj) |
| if obj.closed: |
| value = ''; position = 0 |
| else: |
| value = obj.getvalue(); position = obj.tell() |
| pickler.save_reduce(_create_stringo, (value, position, \ |
| obj.closed), obj=obj) |
| logger.trace(pickler, "# Io") |
| return |
| |
| if LRUCacheType is not None: |
| from functools import lru_cache |
| @register(LRUCacheType) |
| def save_lru_cache(pickler, obj): |
| logger.trace(pickler, "LRU: %s", obj) |
| if OLD39: |
| kwargs = obj.cache_info() |
| args = (kwargs.maxsize,) |
| else: |
| kwargs = obj.cache_parameters() |
| args = (kwargs['maxsize'], kwargs['typed']) |
| if args != lru_cache.__defaults__: |
| wrapper = Reduce(lru_cache, args, is_callable=True) |
| else: |
| wrapper = lru_cache |
| pickler.save_reduce(wrapper, (obj.__wrapped__,), obj=obj) |
| logger.trace(pickler, "# LRU") |
| return |
| |
| @register(SuperType) |
| def save_super(pickler, obj): |
| logger.trace(pickler, "Su: %s", obj) |
| pickler.save_reduce(super, (obj.__thisclass__, obj.__self__), obj=obj) |
| logger.trace(pickler, "# Su") |
| return |
| |
| if IS_PYPY: |
| @register(MethodType) |
| def save_instancemethod0(pickler, obj): |
| code = getattr(obj.__func__, '__code__', None) |
| if code is not None and type(code) is not CodeType \ |
| and getattr(obj.__self__, obj.__name__) == obj: |
| # Some PyPy builtin functions have no module name |
| logger.trace(pickler, "Me2: %s", obj) |
| # TODO: verify that this works for all PyPy builtin methods |
| pickler.save_reduce(getattr, (obj.__self__, obj.__name__), obj=obj) |
| logger.trace(pickler, "# Me2") |
| return |
| |
| logger.trace(pickler, "Me1: %s", obj) |
| pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) |
| logger.trace(pickler, "# Me1") |
| return |
| else: |
| @register(MethodType) |
| def save_instancemethod0(pickler, obj): |
| logger.trace(pickler, "Me1: %s", obj) |
| pickler.save_reduce(MethodType, (obj.__func__, obj.__self__), obj=obj) |
| logger.trace(pickler, "# Me1") |
| return |
| |
| if not IS_PYPY: |
| @register(MemberDescriptorType) |
| @register(GetSetDescriptorType) |
| @register(MethodDescriptorType) |
| @register(WrapperDescriptorType) |
| @register(ClassMethodDescriptorType) |
| def save_wrapper_descriptor(pickler, obj): |
| logger.trace(pickler, "Wr: %s", obj) |
| pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, |
| obj.__repr__()), obj=obj) |
| logger.trace(pickler, "# Wr") |
| return |
| else: |
| @register(MemberDescriptorType) |
| @register(GetSetDescriptorType) |
| def save_wrapper_descriptor(pickler, obj): |
| logger.trace(pickler, "Wr: %s", obj) |
| pickler.save_reduce(_getattr, (obj.__objclass__, obj.__name__, |
| obj.__repr__()), obj=obj) |
| logger.trace(pickler, "# Wr") |
| return |
| |
| @register(CellType) |
| def save_cell(pickler, obj): |
| try: |
| f = obj.cell_contents |
| except ValueError: # cell is empty |
| logger.trace(pickler, "Ce3: %s", obj) |
| # _shims._CELL_EMPTY is defined in _shims.py to support PyPy 2.7. |
| # It unpickles to a sentinel object _dill._CELL_EMPTY, also created in |
| # _shims.py. This object is not present in Python 3 because the cell's |
| # contents can be deleted in newer versions of Python. The reduce object |
| # will instead unpickle to None if unpickled in Python 3. |
| |
| # When breaking changes are made to dill, (_shims._CELL_EMPTY,) can |
| # be replaced by () OR the delattr function can be removed repending on |
| # whichever is more convienient. |
| pickler.save_reduce(_create_cell, (_shims._CELL_EMPTY,), obj=obj) |
| # Call the function _delattr on the cell's cell_contents attribute |
| # The result of this function call will be None |
| pickler.save_reduce(_shims._delattr, (obj, 'cell_contents')) |
| # pop None created by calling _delattr off stack |
| pickler.write(POP) |
| logger.trace(pickler, "# Ce3") |
| return |
| if is_dill(pickler, child=True): |
| if id(f) in pickler._postproc: |
| # Already seen. Add to its postprocessing. |
| postproc = pickler._postproc[id(f)] |
| else: |
| # Haven't seen it. Add to the highest possible object and set its |
| # value as late as possible to prevent cycle. |
| postproc = next(iter(pickler._postproc.values()), None) |
| if postproc is not None: |
| logger.trace(pickler, "Ce2: %s", obj) |
| # _CELL_REF is defined in _shims.py to support older versions of |
| # dill. When breaking changes are made to dill, (_CELL_REF,) can |
| # be replaced by () |
| pickler.save_reduce(_create_cell, (_CELL_REF,), obj=obj) |
| postproc.append((_shims._setattr, (obj, 'cell_contents', f))) |
| logger.trace(pickler, "# Ce2") |
| return |
| logger.trace(pickler, "Ce1: %s", obj) |
| pickler.save_reduce(_create_cell, (f,), obj=obj) |
| logger.trace(pickler, "# Ce1") |
| return |
| |
| if MAPPING_PROXY_TRICK: |
| @register(DictProxyType) |
| def save_dictproxy(pickler, obj): |
| logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj |
| mapping = obj | _dictproxy_helper_instance |
| pickler.save_reduce(DictProxyType, (mapping,), obj=obj) |
| logger.trace(pickler, "# Mp") |
| return |
| else: |
| @register(DictProxyType) |
| def save_dictproxy(pickler, obj): |
| logger.trace(pickler, "Mp: %s", _repr_dict(obj)) # obj |
| pickler.save_reduce(DictProxyType, (obj.copy(),), obj=obj) |
| logger.trace(pickler, "# Mp") |
| return |
| |
| @register(SliceType) |
| def save_slice(pickler, obj): |
| logger.trace(pickler, "Sl: %s", obj) |
| pickler.save_reduce(slice, (obj.start, obj.stop, obj.step), obj=obj) |
| logger.trace(pickler, "# Sl") |
| return |
| |
| @register(XRangeType) |
| @register(EllipsisType) |
| @register(NotImplementedType) |
| def save_singleton(pickler, obj): |
| logger.trace(pickler, "Si: %s", obj) |
| pickler.save_reduce(_eval_repr, (obj.__repr__(),), obj=obj) |
| logger.trace(pickler, "# Si") |
| return |
| |
| def _proxy_helper(obj): # a dead proxy returns a reference to None |
| """get memory address of proxy's reference object""" |
| _repr = repr(obj) |
| try: _str = str(obj) |
| except ReferenceError: # it's a dead proxy |
| return id(None) |
| if _str == _repr: return id(obj) # it's a repr |
| try: # either way, it's a proxy from here |
| address = int(_str.rstrip('>').split(' at ')[-1], base=16) |
| except ValueError: # special case: proxy of a 'type' |
| if not IS_PYPY: |
| address = int(_repr.rstrip('>').split(' at ')[-1], base=16) |
| else: |
| objects = iter(gc.get_objects()) |
| for _obj in objects: |
| if repr(_obj) == _str: return id(_obj) |
| # all bad below... nothing found so throw ReferenceError |
| msg = "Cannot reference object for proxy at '%s'" % id(obj) |
| raise ReferenceError(msg) |
| return address |
| |
| def _locate_object(address, module=None): |
| """get object located at the given memory address (inverse of id(obj))""" |
| special = [None, True, False] #XXX: more...? |
| for obj in special: |
| if address == id(obj): return obj |
| if module: |
| objects = iter(module.__dict__.values()) |
| else: objects = iter(gc.get_objects()) |
| for obj in objects: |
| if address == id(obj): return obj |
| # all bad below... nothing found so throw ReferenceError or TypeError |
| try: address = hex(address) |
| except TypeError: |
| raise TypeError("'%s' is not a valid memory address" % str(address)) |
| raise ReferenceError("Cannot reference object at '%s'" % address) |
| |
| @register(ReferenceType) |
| def save_weakref(pickler, obj): |
| refobj = obj() |
| logger.trace(pickler, "R1: %s", obj) |
| #refobj = ctypes.pythonapi.PyWeakref_GetObject(obj) # dead returns "None" |
| pickler.save_reduce(_create_weakref, (refobj,), obj=obj) |
| logger.trace(pickler, "# R1") |
| return |
| |
| @register(ProxyType) |
| @register(CallableProxyType) |
| def save_weakproxy(pickler, obj): |
| # Must do string substitution here and use %r to avoid ReferenceError. |
| logger.trace(pickler, "R2: %r" % obj) |
| refobj = _locate_object(_proxy_helper(obj)) |
| pickler.save_reduce(_create_weakproxy, (refobj, callable(obj)), obj=obj) |
| logger.trace(pickler, "# R2") |
| return |
| |
| def _is_builtin_module(module): |
| if not hasattr(module, "__file__"): return True |
| if module.__file__ is None: return False |
| # If a module file name starts with prefix, it should be a builtin |
| # module, so should always be pickled as a reference. |
| names = ["base_prefix", "base_exec_prefix", "exec_prefix", "prefix", "real_prefix"] |
| rp = os.path.realpath |
| # See https://github.com/uqfoundation/dill/issues/566 |
| return ( |
| any( |
| module.__file__.startswith(getattr(sys, name)) |
| or rp(module.__file__).startswith(rp(getattr(sys, name))) |
| for name in names |
| if hasattr(sys, name) |
| ) |
| or module.__file__.endswith(EXTENSION_SUFFIXES) |
| or 'site-packages' in module.__file__ |
| ) |
| |
| def _is_imported_module(module): |
| return getattr(module, '__loader__', None) is not None or module in sys.modules.values() |
| |
| @register(ModuleType) |
| def save_module(pickler, obj): |
| if False: #_use_diff: |
| if obj.__name__.split('.', 1)[0] != "dill": |
| try: |
| changed = diff.whats_changed(obj, seen=pickler._diff_cache)[0] |
| except RuntimeError: # not memorised module, probably part of dill |
| pass |
| else: |
| logger.trace(pickler, "M2: %s with diff", obj) |
| logger.info("Diff: %s", changed.keys()) |
| pickler.save_reduce(_import_module, (obj.__name__,), obj=obj, |
| state=changed) |
| logger.trace(pickler, "# M2") |
| return |
| |
| logger.trace(pickler, "M1: %s", obj) |
| pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) |
| logger.trace(pickler, "# M1") |
| else: |
| builtin_mod = _is_builtin_module(obj) |
| is_session_main = is_dill(pickler, child=True) and obj is pickler._main |
| if (obj.__name__ not in ("builtins", "dill", "dill._dill") and not builtin_mod |
| or is_session_main): |
| logger.trace(pickler, "M1: %s", obj) |
| # Hack for handling module-type objects in load_module(). |
| mod_name = obj.__name__ if _is_imported_module(obj) else '__runtime__.%s' % obj.__name__ |
| # Second references are saved as __builtin__.__main__ in save_module_dict(). |
| main_dict = obj.__dict__.copy() |
| for item in ('__builtins__', '__loader__'): |
| main_dict.pop(item, None) |
| for item in IPYTHON_SINGLETONS: #pragma: no cover |
| if getattr(main_dict.get(item), '__module__', '').startswith('IPython'): |
| del main_dict[item] |
| pickler.save_reduce(_import_module, (mod_name,), obj=obj, state=main_dict) |
| logger.trace(pickler, "# M1") |
| elif obj.__name__ == "dill._dill": |
| logger.trace(pickler, "M2: %s", obj) |
| pickler.save_global(obj, name="_dill") |
| logger.trace(pickler, "# M2") |
| else: |
| logger.trace(pickler, "M2: %s", obj) |
| pickler.save_reduce(_import_module, (obj.__name__,), obj=obj) |
| logger.trace(pickler, "# M2") |
| return |
| |
| # The following function is based on '_extract_class_dict' from 'cloudpickle' |
| # Copyright (c) 2012, Regents of the University of California. |
| # Copyright (c) 2009 `PiCloud, Inc. <http://www.picloud.com>`_. |
| # License: https://github.com/cloudpipe/cloudpickle/blob/master/LICENSE |
| def _get_typedict_type(cls, clsdict, attrs, postproc_list): |
| """Retrieve a copy of the dict of a class without the inherited methods""" |
| if len(cls.__bases__) == 1: |
| inherited_dict = cls.__bases__[0].__dict__ |
| else: |
| inherited_dict = {} |
| for base in reversed(cls.__bases__): |
| inherited_dict.update(base.__dict__) |
| to_remove = [] |
| for name, value in dict.items(clsdict): |
| try: |
| base_value = inherited_dict[name] |
| if value is base_value and hasattr(value, '__qualname__'): |
| to_remove.append(name) |
| except KeyError: |
| pass |
| for name in to_remove: |
| dict.pop(clsdict, name) |
| |
| if issubclass(type(cls), type): |
| clsdict.pop('__dict__', None) |
| clsdict.pop('__weakref__', None) |
| # clsdict.pop('__prepare__', None) |
| return clsdict, attrs |
| |
| def _get_typedict_abc(obj, _dict, attrs, postproc_list): |
| if hasattr(abc, '_get_dump'): |
| (registry, _, _, _) = abc._get_dump(obj) |
| register = obj.register |
| postproc_list.extend((register, (reg(),)) for reg in registry) |
| elif hasattr(obj, '_abc_registry'): |
| registry = obj._abc_registry |
| register = obj.register |
| postproc_list.extend((register, (reg,)) for reg in registry) |
| else: |
| raise PicklingError("Cannot find registry of ABC %s", obj) |
| |
| if '_abc_registry' in _dict: |
| _dict.pop('_abc_registry', None) |
| _dict.pop('_abc_cache', None) |
| _dict.pop('_abc_negative_cache', None) |
| # _dict.pop('_abc_negative_cache_version', None) |
| else: |
| _dict.pop('_abc_impl', None) |
| return _dict, attrs |
| |
| @register(TypeType) |
| def save_type(pickler, obj, postproc_list=None): |
| if obj in _typemap: |
| logger.trace(pickler, "T1: %s", obj) |
| # if obj in _incedental_types: |
| # warnings.warn('Type %r may only exist on this implementation of Python and cannot be unpickled in other implementations.' % (obj,), PicklingWarning) |
| pickler.save_reduce(_load_type, (_typemap[obj],), obj=obj) |
| logger.trace(pickler, "# T1") |
| elif obj.__bases__ == (tuple,) and all([hasattr(obj, attr) for attr in ('_fields','_asdict','_make','_replace')]): |
| # special case: namedtuples |
| logger.trace(pickler, "T6: %s", obj) |
| |
| obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) |
| if obj.__name__ != obj_name: |
| if postproc_list is None: |
| postproc_list = [] |
| postproc_list.append((setattr, (obj, '__qualname__', obj_name))) |
| |
| if not obj._field_defaults: |
| _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__)), obj=obj, postproc_list=postproc_list) |
| else: |
| defaults = [obj._field_defaults[field] for field in obj._fields if field in obj._field_defaults] |
| _save_with_postproc(pickler, (_create_namedtuple, (obj.__name__, obj._fields, obj.__module__, defaults)), obj=obj, postproc_list=postproc_list) |
| logger.trace(pickler, "# T6") |
| return |
| |
| # special cases: NoneType, NotImplementedType, EllipsisType, EnumMeta |
| elif obj is type(None): |
| logger.trace(pickler, "T7: %s", obj) |
| #XXX: pickler.save_reduce(type, (None,), obj=obj) |
| pickler.write(GLOBAL + b'__builtin__\nNoneType\n') |
| logger.trace(pickler, "# T7") |
| elif obj is NotImplementedType: |
| logger.trace(pickler, "T7: %s", obj) |
| pickler.save_reduce(type, (NotImplemented,), obj=obj) |
| logger.trace(pickler, "# T7") |
| elif obj is EllipsisType: |
| logger.trace(pickler, "T7: %s", obj) |
| pickler.save_reduce(type, (Ellipsis,), obj=obj) |
| logger.trace(pickler, "# T7") |
| elif obj is EnumMeta: |
| logger.trace(pickler, "T7: %s", obj) |
| pickler.write(GLOBAL + b'enum\nEnumMeta\n') |
| logger.trace(pickler, "# T7") |
| |
| else: |
| _byref = getattr(pickler, '_byref', None) |
| obj_recursive = id(obj) in getattr(pickler, '_postproc', ()) |
| incorrectly_named = not _locate_function(obj, pickler) |
| if not _byref and not obj_recursive and incorrectly_named: # not a function, but the name was held over |
| if postproc_list is None: |
| postproc_list = [] |
| |
| # thanks to Tom Stepleton pointing out pickler._session unneeded |
| logger.trace(pickler, "T2: %s", obj) |
| _dict, attrs = _get_typedict_type(obj, obj.__dict__.copy(), None, postproc_list) # copy dict proxy to a dict |
| |
| #print (_dict) |
| #print ("%s\n%s" % (type(obj), obj.__name__)) |
| #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) |
| slots = _dict.get('__slots__', ()) |
| if type(slots) == str: |
| # __slots__ accepts a single string |
| slots = (slots,) |
| |
| for name in slots: |
| _dict.pop(name, None) |
| |
| if isinstance(obj, abc.ABCMeta): |
| logger.trace(pickler, "ABC: %s", obj) |
| _dict, attrs = _get_typedict_abc(obj, _dict, attrs, postproc_list) |
| logger.trace(pickler, "# ABC") |
| |
| qualname = getattr(obj, '__qualname__', None) |
| if attrs is not None: |
| for k, v in attrs.items(): |
| postproc_list.append((setattr, (obj, k, v))) |
| # TODO: Consider using the state argument to save_reduce? |
| if qualname is not None: |
| postproc_list.append((setattr, (obj, '__qualname__', qualname))) |
| |
| if not hasattr(obj, '__orig_bases__'): |
| _save_with_postproc(pickler, (_create_type, ( |
| type(obj), obj.__name__, obj.__bases__, _dict |
| )), obj=obj, postproc_list=postproc_list) |
| else: |
| # This case will always work, but might be overkill. |
| _metadict = { |
| 'metaclass': type(obj) |
| } |
| |
| if _dict: |
| _dict_update = PartialType(_setitems, source=_dict) |
| else: |
| _dict_update = None |
| |
| _save_with_postproc(pickler, (new_class, ( |
| obj.__name__, obj.__orig_bases__, _metadict, _dict_update |
| )), obj=obj, postproc_list=postproc_list) |
| logger.trace(pickler, "# T2") |
| else: |
| obj_name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) |
| logger.trace(pickler, "T4: %s", obj) |
| if incorrectly_named: |
| warnings.warn( |
| "Cannot locate reference to %r." % (obj,), |
| PicklingWarning, |
| stacklevel=3, |
| ) |
| if obj_recursive: |
| warnings.warn( |
| "Cannot pickle %r: %s.%s has recursive self-references that " |
| "trigger a RecursionError." % (obj, obj.__module__, obj_name), |
| PicklingWarning, |
| stacklevel=3, |
| ) |
| #print (obj.__dict__) |
| #print ("%s\n%s" % (type(obj), obj.__name__)) |
| #print ("%s\n%s" % (obj.__bases__, obj.__dict__)) |
| StockPickler.save_global(pickler, obj, name=obj_name) |
| logger.trace(pickler, "# T4") |
| return |
| |
| @register(property) |
| @register(abc.abstractproperty) |
| def save_property(pickler, obj): |
| logger.trace(pickler, "Pr: %s", obj) |
| pickler.save_reduce(type(obj), (obj.fget, obj.fset, obj.fdel, obj.__doc__), |
| obj=obj) |
| logger.trace(pickler, "# Pr") |
| |
| @register(staticmethod) |
| @register(classmethod) |
| @register(abc.abstractstaticmethod) |
| @register(abc.abstractclassmethod) |
| def save_classmethod(pickler, obj): |
| logger.trace(pickler, "Cm: %s", obj) |
| orig_func = obj.__func__ |
| |
| # if type(obj.__dict__) is dict: |
| # if obj.__dict__: |
| # state = obj.__dict__ |
| # else: |
| # state = None |
| # else: |
| # state = (None, {'__dict__', obj.__dict__}) |
| |
| pickler.save_reduce(type(obj), (orig_func,), obj=obj) |
| logger.trace(pickler, "# Cm") |
| |
| @register(FunctionType) |
| def save_function(pickler, obj): |
| if not _locate_function(obj, pickler): |
| if type(obj.__code__) is not CodeType: |
| # Some PyPy builtin functions have no module name, and thus are not |
| # able to be located |
| module_name = getattr(obj, '__module__', None) |
| if module_name is None: |
| module_name = __builtin__.__name__ |
| module = _import_module(module_name, safe=True) |
| _pypy_builtin = False |
| try: |
| found, _ = _getattribute(module, obj.__qualname__) |
| if getattr(found, '__func__', None) is obj: |
| _pypy_builtin = True |
| except AttributeError: |
| pass |
| |
| if _pypy_builtin: |
| logger.trace(pickler, "F3: %s", obj) |
| pickler.save_reduce(getattr, (found, '__func__'), obj=obj) |
| logger.trace(pickler, "# F3") |
| return |
| |
| logger.trace(pickler, "F1: %s", obj) |
| _recurse = getattr(pickler, '_recurse', None) |
| _postproc = getattr(pickler, '_postproc', None) |
| _main_modified = getattr(pickler, '_main_modified', None) |
| _original_main = getattr(pickler, '_original_main', __builtin__)#'None' |
| postproc_list = [] |
| if _recurse: |
| # recurse to get all globals referred to by obj |
| from .detect import globalvars |
| globs_copy = globalvars(obj, recurse=True, builtin=True) |
| |
| # Add the name of the module to the globs dictionary to prevent |
| # the duplication of the dictionary. Pickle the unpopulated |
| # globals dictionary and set the remaining items after the function |
| # is created to correctly handle recursion. |
| globs = {'__name__': obj.__module__} |
| else: |
| globs_copy = obj.__globals__ |
| |
| # If the globals is the __dict__ from the module being saved as a |
| # session, substitute it by the dictionary being actually saved. |
| if _main_modified and globs_copy is _original_main.__dict__: |
| globs_copy = getattr(pickler, '_main', _original_main).__dict__ |
| globs = globs_copy |
| # If the globals is a module __dict__, do not save it in the pickle. |
| elif globs_copy is not None and obj.__module__ is not None and \ |
| getattr(_import_module(obj.__module__, True), '__dict__', None) is globs_copy: |
| globs = globs_copy |
| else: |
| globs = {'__name__': obj.__module__} |
| |
| if globs_copy is not None and globs is not globs_copy: |
| # In the case that the globals are copied, we need to ensure that |
| # the globals dictionary is updated when all objects in the |
| # dictionary are already created. |
| glob_ids = {id(g) for g in globs_copy.values()} |
| for stack_element in _postproc: |
| if stack_element in glob_ids: |
| _postproc[stack_element].append((_setitems, (globs, globs_copy))) |
| break |
| else: |
| postproc_list.append((_setitems, (globs, globs_copy))) |
| |
| closure = obj.__closure__ |
| state_dict = {} |
| for fattrname in ('__doc__', '__kwdefaults__', '__annotations__'): |
| fattr = getattr(obj, fattrname, None) |
| if fattr is not None: |
| state_dict[fattrname] = fattr |
| if obj.__qualname__ != obj.__name__: |
| state_dict['__qualname__'] = obj.__qualname__ |
| if '__name__' not in globs or obj.__module__ != globs['__name__']: |
| state_dict['__module__'] = obj.__module__ |
| |
| state = obj.__dict__ |
| if type(state) is not dict: |
| state_dict['__dict__'] = state |
| state = None |
| if state_dict: |
| state = state, state_dict |
| |
| _save_with_postproc(pickler, (_create_function, ( |
| obj.__code__, globs, obj.__name__, obj.__defaults__, |
| closure |
| ), state), obj=obj, postproc_list=postproc_list) |
| |
| # Lift closure cell update to earliest function (#458) |
| if _postproc: |
| topmost_postproc = next(iter(_postproc.values()), None) |
| if closure and topmost_postproc: |
| for cell in closure: |
| possible_postproc = (setattr, (cell, 'cell_contents', obj)) |
| try: |
| topmost_postproc.remove(possible_postproc) |
| except ValueError: |
| continue |
| |
| # Change the value of the cell |
| pickler.save_reduce(*possible_postproc) |
| # pop None created by calling preprocessing step off stack |
| pickler.write(POP) |
| |
| logger.trace(pickler, "# F1") |
| else: |
| logger.trace(pickler, "F2: %s", obj) |
| name = getattr(obj, '__qualname__', getattr(obj, '__name__', None)) |
| StockPickler.save_global(pickler, obj, name=name) |
| logger.trace(pickler, "# F2") |
| return |
| |
| if HAS_CTYPES and hasattr(ctypes, 'pythonapi'): |
| _PyCapsule_New = ctypes.pythonapi.PyCapsule_New |
| _PyCapsule_New.argtypes = (ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p) |
| _PyCapsule_New.restype = ctypes.py_object |
| _PyCapsule_GetPointer = ctypes.pythonapi.PyCapsule_GetPointer |
| _PyCapsule_GetPointer.argtypes = (ctypes.py_object, ctypes.c_char_p) |
| _PyCapsule_GetPointer.restype = ctypes.c_void_p |
| _PyCapsule_GetDestructor = ctypes.pythonapi.PyCapsule_GetDestructor |
| _PyCapsule_GetDestructor.argtypes = (ctypes.py_object,) |
| _PyCapsule_GetDestructor.restype = ctypes.c_void_p |
| _PyCapsule_GetContext = ctypes.pythonapi.PyCapsule_GetContext |
| _PyCapsule_GetContext.argtypes = (ctypes.py_object,) |
| _PyCapsule_GetContext.restype = ctypes.c_void_p |
| _PyCapsule_GetName = ctypes.pythonapi.PyCapsule_GetName |
| _PyCapsule_GetName.argtypes = (ctypes.py_object,) |
| _PyCapsule_GetName.restype = ctypes.c_char_p |
| _PyCapsule_IsValid = ctypes.pythonapi.PyCapsule_IsValid |
| _PyCapsule_IsValid.argtypes = (ctypes.py_object, ctypes.c_char_p) |
| _PyCapsule_IsValid.restype = ctypes.c_bool |
| _PyCapsule_SetContext = ctypes.pythonapi.PyCapsule_SetContext |
| _PyCapsule_SetContext.argtypes = (ctypes.py_object, ctypes.c_void_p) |
| _PyCapsule_SetDestructor = ctypes.pythonapi.PyCapsule_SetDestructor |
| _PyCapsule_SetDestructor.argtypes = (ctypes.py_object, ctypes.c_void_p) |
| _PyCapsule_SetName = ctypes.pythonapi.PyCapsule_SetName |
| _PyCapsule_SetName.argtypes = (ctypes.py_object, ctypes.c_char_p) |
| _PyCapsule_SetPointer = ctypes.pythonapi.PyCapsule_SetPointer |
| _PyCapsule_SetPointer.argtypes = (ctypes.py_object, ctypes.c_void_p) |
| #from _socket import CAPI as _testcapsule |
| _testcapsule_name = b'dill._dill._testcapsule' |
| _testcapsule = _PyCapsule_New( |
| ctypes.cast(_PyCapsule_New, ctypes.c_void_p), |
| ctypes.c_char_p(_testcapsule_name), |
| None |
| ) |
| PyCapsuleType = type(_testcapsule) |
| @register(PyCapsuleType) |
| def save_capsule(pickler, obj): |
| logger.trace(pickler, "Cap: %s", obj) |
| name = _PyCapsule_GetName(obj) |
| #warnings.warn('Pickling a PyCapsule (%s) does not pickle any C data structures and could cause segmentation faults or other memory errors when unpickling.' % (name,), PicklingWarning) |
| pointer = _PyCapsule_GetPointer(obj, name) |
| context = _PyCapsule_GetContext(obj) |
| destructor = _PyCapsule_GetDestructor(obj) |
| pickler.save_reduce(_create_capsule, (pointer, name, context, destructor), obj=obj) |
| logger.trace(pickler, "# Cap") |
| _incedental_reverse_typemap['PyCapsuleType'] = PyCapsuleType |
| _reverse_typemap['PyCapsuleType'] = PyCapsuleType |
| _incedental_types.add(PyCapsuleType) |
| else: |
| _testcapsule = None |
| |
| |
| ############################# |
| # A quick fix for issue #500 |
| # This should be removed when a better solution is found. |
| |
| if hasattr(dataclasses, "_HAS_DEFAULT_FACTORY_CLASS"): |
| @register(dataclasses._HAS_DEFAULT_FACTORY_CLASS) |
| def save_dataclasses_HAS_DEFAULT_FACTORY_CLASS(pickler, obj): |
| logger.trace(pickler, "DcHDF: %s", obj) |
| pickler.write(GLOBAL + b"dataclasses\n_HAS_DEFAULT_FACTORY\n") |
| logger.trace(pickler, "# DcHDF") |
| |
| if hasattr(dataclasses, "MISSING"): |
| @register(type(dataclasses.MISSING)) |
| def save_dataclasses_MISSING_TYPE(pickler, obj): |
| logger.trace(pickler, "DcM: %s", obj) |
| pickler.write(GLOBAL + b"dataclasses\nMISSING\n") |
| logger.trace(pickler, "# DcM") |
| |
| if hasattr(dataclasses, "KW_ONLY"): |
| @register(type(dataclasses.KW_ONLY)) |
| def save_dataclasses_KW_ONLY_TYPE(pickler, obj): |
| logger.trace(pickler, "DcKWO: %s", obj) |
| pickler.write(GLOBAL + b"dataclasses\nKW_ONLY\n") |
| logger.trace(pickler, "# DcKWO") |
| |
| if hasattr(dataclasses, "_FIELD_BASE"): |
| @register(dataclasses._FIELD_BASE) |
| def save_dataclasses_FIELD_BASE(pickler, obj): |
| logger.trace(pickler, "DcFB: %s", obj) |
| pickler.write(GLOBAL + b"dataclasses\n" + obj.name.encode() + b"\n") |
| logger.trace(pickler, "# DcFB") |
| |
| ############################# |
| |
| # quick sanity checking |
| def pickles(obj,exact=False,safe=False,**kwds): |
| """ |
| Quick check if object pickles with dill. |
| |
| If *exact=True* then an equality test is done to check if the reconstructed |
| object matches the original object. |
| |
| If *safe=True* then any exception will raised in copy signal that the |
| object is not picklable, otherwise only pickling errors will be trapped. |
| |
| Additional keyword arguments are as :func:`dumps` and :func:`loads`. |
| """ |
| if safe: exceptions = (Exception,) # RuntimeError, ValueError |
| else: |
| exceptions = (TypeError, AssertionError, NotImplementedError, PicklingError, UnpicklingError) |
| try: |
| pik = copy(obj, **kwds) |
| #FIXME: should check types match first, then check content if "exact" |
| try: |
| #FIXME: should be "(pik == obj).all()" for numpy comparison, though that'll fail if shapes differ |
| result = bool(pik.all() == obj.all()) |
| except (AttributeError, TypeError): |
| warnings.filterwarnings('ignore') #FIXME: be specific |
| result = pik == obj |
| if warnings.filters: del warnings.filters[0] |
| if hasattr(result, 'toarray'): # for unusual types like sparse matrix |
| result = result.toarray().all() |
| if result: return True |
| if not exact: |
| result = type(pik) == type(obj) |
| if result: return result |
| # class instances might have been dumped with byref=False |
| return repr(type(pik)) == repr(type(obj)) #XXX: InstanceType? |
| return False |
| except exceptions: |
| return False |
| |
| def check(obj, *args, **kwds): |
| """ |
| Check pickling of an object across another process. |
| |
| *python* is the path to the python interpreter (defaults to sys.executable) |
| |
| Set *verbose=True* to print the unpickled object in the other process. |
| |
| Additional keyword arguments are as :func:`dumps` and :func:`loads`. |
| """ |
| # == undocumented == |
| # python -- the string path or executable name of the selected python |
| # verbose -- if True, be verbose about printing warning messages |
| # all other args and kwds are passed to dill.dumps #FIXME: ignore on load |
| verbose = kwds.pop('verbose', False) |
| python = kwds.pop('python', None) |
| if python is None: |
| import sys |
| python = sys.executable |
| # type check |
| isinstance(python, str) |
| import subprocess |
| fail = True |
| try: |
| _obj = dumps(obj, *args, **kwds) |
| fail = False |
| finally: |
| if fail and verbose: |
| print("DUMP FAILED") |
| #FIXME: fails if python interpreter path contains spaces |
| # Use the following instead (which also processes the 'ignore' keyword): |
| # ignore = kwds.pop('ignore', None) |
| # unpickle = "dill.loads(%s, ignore=%s)"%(repr(_obj), repr(ignore)) |
| # cmd = [python, "-c", "import dill; print(%s)"%unpickle] |
| # msg = "SUCCESS" if not subprocess.call(cmd) else "LOAD FAILED" |
| msg = "%s -c import dill; print(dill.loads(%s))" % (python, repr(_obj)) |
| msg = "SUCCESS" if not subprocess.call(msg.split(None,2)) else "LOAD FAILED" |
| if verbose: |
| print(msg) |
| return |
| |
| # use to protect against missing attributes |
| def is_dill(pickler, child=None): |
| "check the dill-ness of your pickler" |
| if child is False or not hasattr(pickler.__class__, 'mro'): |
| return 'dill' in pickler.__module__ |
| return Pickler in pickler.__class__.mro() |
| |
| def _extend(): |
| """extend pickle with all of dill's registered types""" |
| # need to have pickle not choke on _main_module? use is_dill(pickler) |
| for t,func in Pickler.dispatch.items(): |
| try: |
| StockPickler.dispatch[t] = func |
| except Exception: #TypeError, PicklingError, UnpicklingError |
| logger.trace(pickler, "skip: %s", t) |
| return |
| |
| del diff, _use_diff, use_diff |
| |
| # EOF |