[Python-modules-commits] [cloudpickle] 05/09: New upstream version 0.4.0
Diane Trout
diane at moszumanska.debian.org
Fri Oct 13 17:25:39 UTC 2017
This is an automated email from the git hooks/post-receive script.
diane pushed a commit to branch master
in repository cloudpickle.
commit 358a2ad6d97311538ed1b721034f715bfdf2dae7
Author: Diane Trout <diane at ghic.org>
Date: Fri Oct 13 08:21:39 2017 -0700
New upstream version 0.4.0
---
PKG-INFO | 2 +-
cloudpickle.egg-info/PKG-INFO | 2 +-
cloudpickle/__init__.py | 2 +-
cloudpickle/cloudpickle.py | 194 +++++++++++++++++++++++++++++++--------
setup.cfg | 1 +
setup.py | 2 +-
tests/cloudpickle_test.py | 209 ++++++++++++++++++++++++++++++++++++++----
7 files changed, 351 insertions(+), 61 deletions(-)
diff --git a/PKG-INFO b/PKG-INFO
index cf7660e..75e76c5 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.3.1
+Version: 0.4.0
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
diff --git a/cloudpickle.egg-info/PKG-INFO b/cloudpickle.egg-info/PKG-INFO
index cf7660e..75e76c5 100644
--- a/cloudpickle.egg-info/PKG-INFO
+++ b/cloudpickle.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.3.1
+Version: 0.4.0
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
diff --git a/cloudpickle/__init__.py b/cloudpickle/__init__.py
index 245645e..0cbfd9e 100644
--- a/cloudpickle/__init__.py
+++ b/cloudpickle/__init__.py
@@ -2,4 +2,4 @@ from __future__ import absolute_import
from cloudpickle.cloudpickle import *
-__version__ = '0.3.1'
+__version__ = '0.4.0'
diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py
index 030d44a..17a1f95 100644
--- a/cloudpickle/cloudpickle.py
+++ b/cloudpickle/cloudpickle.py
@@ -47,6 +47,7 @@ from functools import partial
import imp
import io
import itertools
+import logging
import opcode
import operator
import pickle
@@ -309,7 +310,12 @@ class CloudPickler(Pickler):
if name is None:
name = obj.__name__
- modname = pickle.whichmodule(obj, name)
+ try:
+ # whichmodule() could fail, see
+ # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
+ modname = pickle.whichmodule(obj, name)
+ except Exception:
+ modname = None
# print('which gives %s %s %s' % (modname, obj, name))
try:
themodule = sys.modules[modname]
@@ -383,7 +389,8 @@ class CloudPickler(Pickler):
# check if the package has any currently loaded sub-imports
prefix = x.__name__ + '.'
for name, module in sys.modules.items():
- if name.startswith(prefix):
+ # Older versions of pytest will add a "None" module to sys.modules.
+ if name is not None and name.startswith(prefix):
# check whether the function can address the sub-module
tokens = set(name[len(prefix):].split('.'))
if not tokens - set(code.co_names):
@@ -392,6 +399,71 @@ class CloudPickler(Pickler):
# then discards the reference to it
self.write(pickle.POP)
+ def save_dynamic_class(self, obj):
+ """
+ Save a class that can't be stored as module global.
+
+ This method is used to serialize classes that are defined inside
+ functions, or that otherwise can't be serialized as attribute lookups
+ from global modules.
+ """
+ clsdict = dict(obj.__dict__) # copy dict proxy to a dict
+ if not isinstance(clsdict.get('__dict__', None), property):
+ # don't extract dict that are properties
+ clsdict.pop('__dict__', None)
+ clsdict.pop('__weakref__', None)
+
+ # hack as __new__ is stored differently in the __dict__
+ new_override = clsdict.get('__new__', None)
+ if new_override:
+ clsdict['__new__'] = obj.__new__
+
+ save = self.save
+ write = self.write
+
+ # We write pickle instructions explicitly here to handle the
+ # possibility that the type object participates in a cycle with its own
+ # __dict__. We first write an empty "skeleton" version of the class and
+ # memoize it before writing the class' __dict__ itself. We then write
+ # instructions to "rehydrate" the skeleton class by restoring the
+ # attributes from the __dict__.
+ #
+ # A type can appear in a cycle with its __dict__ if an instance of the
+ # type appears in the type's __dict__ (which happens for the stdlib
+ # Enum class), or if the type defines methods that close over the name
+ # of the type, (which is common for Python 2-style super() calls).
+
+ # Push the rehydration function.
+ save(_rehydrate_skeleton_class)
+
+ # Mark the start of the args for the rehydration function.
+ write(pickle.MARK)
+
+ # On PyPy, __doc__ is a readonly attribute, so we need to include it in
+ # the initial skeleton class. This is safe because we know that the
+ # doc can't participate in a cycle with the original class.
+ doc_dict = {'__doc__': clsdict.pop('__doc__', None)}
+
+ # Create and memoize an empty class with obj's name and bases.
+ save(type(obj))
+ save((
+ obj.__name__,
+ obj.__bases__,
+ doc_dict,
+ ))
+ write(pickle.REDUCE)
+ self.memoize(obj)
+
+ # Now save the rest of obj's __dict__. Any references to obj
+ # encountered while saving will point to the skeleton class.
+ save(clsdict)
+
+ # Write a tuple of (skeleton_class, clsdict).
+ write(pickle.TUPLE)
+
+ # Call _rehydrate_skeleton_class(skeleton_class, clsdict)
+ write(pickle.REDUCE)
+
def save_function_tuple(self, func):
""" Pickles an actual func object.
@@ -492,8 +564,9 @@ class CloudPickler(Pickler):
# process closure
closure = (
- [c.cell_contents for c in func.__closure__]
- if func.__closure__ is not None else None
+ list(map(_get_cell_contents, func.__closure__))
+ if func.__closure__ is not None
+ else None
)
# save the dict
@@ -511,6 +584,12 @@ class CloudPickler(Pickler):
dispatch[types.BuiltinFunctionType] = save_builtin_function
def save_global(self, obj, name=None, pack=struct.pack):
+ """
+ Save a "global".
+
+ The name of this method is somewhat misleading: all types get
+ dispatched here.
+ """
if obj.__module__ == "__builtin__" or obj.__module__ == "builtins":
if obj in _BUILTIN_TYPE_NAMES:
return self.save_reduce(_builtin_type, (_BUILTIN_TYPE_NAMES[obj],), obj=obj)
@@ -520,7 +599,12 @@ class CloudPickler(Pickler):
modname = getattr(obj, "__module__", None)
if modname is None:
- modname = pickle.whichmodule(obj, name)
+ try:
+ # whichmodule() could fail, see
+ # https://bitbucket.org/gutworth/six/issues/63/importing-six-breaks-pickling
+ modname = pickle.whichmodule(obj, name)
+ except Exception:
+ modname = '__main__'
if modname == '__main__':
themodule = None
@@ -534,18 +618,7 @@ class CloudPickler(Pickler):
typ = type(obj)
if typ is not obj and isinstance(obj, (type, types.ClassType)):
- d = dict(obj.__dict__) # copy dict proxy to a dict
- if not isinstance(d.get('__dict__', None), property):
- # don't extract dict that are properties
- d.pop('__dict__', None)
- d.pop('__weakref__', None)
-
- # hack as __new__ is stored differently in the __dict__
- new_override = d.get('__new__', None)
- if new_override:
- d['__new__'] = obj.__new__
-
- self.save_reduce(typ, (obj.__name__, obj.__bases__, d), obj=obj)
+ self.save_dynamic_class(obj)
else:
raise pickle.PicklingError("Can't pickle %r" % obj)
@@ -565,10 +638,15 @@ class CloudPickler(Pickler):
dispatch[types.MethodType] = save_instancemethod
def save_inst(self, obj):
- """Inner logic to save instance. Based off pickle.save_inst
- Supports __transient__"""
+ """Inner logic to save instance. Based off pickle.save_inst"""
cls = obj.__class__
+ # Try the dispatch table (pickle module doesn't do it)
+ f = self.dispatch.get(cls)
+ if f:
+ f(self, obj) # Call unbound method with explicit self
+ return
+
memo = self.memo
write = self.write
save = self.save
@@ -598,13 +676,6 @@ class CloudPickler(Pickler):
getstate = obj.__getstate__
except AttributeError:
stuff = obj.__dict__
- #remove items if transient
- if hasattr(obj, '__transient__'):
- transient = obj.__transient__
- stuff = stuff.copy()
- for k in list(stuff.keys()):
- if k in transient:
- del stuff[k]
else:
stuff = getstate()
pickle._keep_alive(stuff, memo)
@@ -667,8 +738,6 @@ class CloudPickler(Pickler):
def save_reduce(self, func, args, state=None,
listitems=None, dictitems=None, obj=None):
- """Modified to support __transient__ on new objects
- Change only affects protocol level 2 (which is always used by PiCloud"""
# Assert that args is a tuple or None
if not isinstance(args, tuple):
raise pickle.PicklingError("args from reduce() should be a tuple")
@@ -682,7 +751,6 @@ class CloudPickler(Pickler):
# Protocol 2 special case: if func's name is __newobj__, use NEWOBJ
if self.proto >= 2 and getattr(func, "__name__", "") == "__newobj__":
- #Added fix to allow transient
cls = args[0]
if not hasattr(cls, "__new__"):
raise pickle.PicklingError(
@@ -693,15 +761,6 @@ class CloudPickler(Pickler):
args = args[1:]
save(cls)
- #Don't pickle transient entries
- if hasattr(obj, '__transient__'):
- transient = obj.__transient__
- state = state.copy()
-
- for k in list(state.keys()):
- if k in transient:
- del state[k]
-
save(args)
write(pickle.NEWOBJ)
else:
@@ -790,11 +849,23 @@ class CloudPickler(Pickler):
dispatch[type(Ellipsis)] = save_ellipsis
dispatch[type(NotImplemented)] = save_not_implemented
+ # WeakSet was added in 2.7.
+ if hasattr(weakref, 'WeakSet'):
+ def save_weakset(self, obj):
+ self.save_reduce(weakref.WeakSet, (list(obj),))
+
+ dispatch[weakref.WeakSet] = save_weakset
+
"""Special functions for Add-on libraries"""
def inject_addons(self):
"""Plug in system. Register additional pickling functions if modules already loaded"""
pass
+ def save_logger(self, obj):
+ self.save_reduce(logging.getLogger, (obj.name,), obj=obj)
+
+ dispatch[logging.Logger] = save_logger
+
# Tornado support
@@ -896,6 +967,40 @@ def _gen_ellipsis():
def _gen_not_implemented():
return NotImplemented
+
+def _get_cell_contents(cell):
+ try:
+ return cell.cell_contents
+ except ValueError:
+ # sentinel used by ``_fill_function`` which will leave the cell empty
+ return _empty_cell_value
+
+
+def instance(cls):
+ """Create a new instance of a class.
+
+ Parameters
+ ----------
+ cls : type
+ The class to create an instance of.
+
+ Returns
+ -------
+ instance : cls
+ A new instance of ``cls``.
+ """
+ return cls()
+
+
+ at instance
+class _empty_cell_value(object):
+ """sentinel for empty closures
+ """
+ @classmethod
+ def __reduce__(cls):
+ return cls.__name__
+
+
def _fill_function(func, globals, defaults, dict, closure_values):
""" Fills in the rest of function data into the skeleton function object
that were created via _make_skel_func().
@@ -907,7 +1012,8 @@ def _fill_function(func, globals, defaults, dict, closure_values):
cells = func.__closure__
if cells is not None:
for cell, value in zip(cells, closure_values):
- cell_set(cell, value)
+ if value is not _empty_cell_value:
+ cell_set(cell, value)
return func
@@ -938,6 +1044,16 @@ def _make_skel_func(code, cell_count, base_globals=None):
return types.FunctionType(code, base_globals, None, None, closure)
+def _rehydrate_skeleton_class(skeleton_class, class_dict):
+ """Put attributes from `class_dict` back on `skeleton_class`.
+
+ See CloudPickler.save_dynamic_class for more info.
+ """
+ for attrname, attr in class_dict.items():
+ setattr(skeleton_class, attrname, attr)
+ return skeleton_class
+
+
def _find_module(mod_name):
"""
Iterate over each part instead of calling imp.find_module directly.
diff --git a/setup.cfg b/setup.cfg
index 1e3eb36..6c71b61 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,4 +4,5 @@ universal = 1
[egg_info]
tag_build =
tag_date = 0
+tag_svn_revision = 0
diff --git a/setup.py b/setup.py
index e59d90b..8987560 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@ except ImportError:
dist = setup(
name='cloudpickle',
- version='0.3.1',
+ version='0.4.0',
description='Extended pickling support for Python objects',
author='Cloudpipe',
author_email='cloudpipe at googlegroups.com',
diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py
index 19f1faf..aa33ce4 100644
--- a/tests/cloudpickle_test.py
+++ b/tests/cloudpickle_test.py
@@ -1,16 +1,29 @@
from __future__ import division
-import imp
-import unittest
-import pytest
-import pickle
-import sys
-import random
+
+import abc
+
+import base64
import functools
+import imp
+from io import BytesIO
import itertools
+import logging
+from operator import itemgetter, attrgetter
+import pickle
import platform
-import textwrap
-import base64
+import random
import subprocess
+import sys
+import textwrap
+import unittest
+import weakref
+
+try:
+ from StringIO import StringIO
+except ImportError:
+ from io import StringIO
+
+import pytest
try:
# try importing numpy and scipy. These are not hard dependencies and
@@ -27,22 +40,15 @@ try:
except ImportError:
tornado = None
-
-from operator import itemgetter, attrgetter
-
-try:
- from StringIO import StringIO
-except ImportError:
- from io import StringIO
-
-from io import BytesIO
-
import cloudpickle
from cloudpickle.cloudpickle import _find_module, _make_empty_cell, cell_set
from .testutils import subprocess_pickle_echo
+HAVE_WEAKSET = hasattr(weakref, 'WeakSet')
+
+
def pickle_depickle(obj):
"""Helper function to test whether object pickled with cloudpickle can be
depickled with pickle
@@ -167,6 +173,24 @@ class CloudPickleTest(unittest.TestCase):
msg='g now has closure cells even though f does not',
)
+ def test_empty_cell_preserved(self):
+ def f():
+ if False: # pragma: no cover
+ cell = None
+
+ def g():
+ cell # NameError, unbound free variable
+
+ return g
+
+ g1 = f()
+ with pytest.raises(NameError):
+ g1()
+
+ g2 = pickle_depickle(g1)
+ with pytest.raises(NameError):
+ g2()
+
def test_unhashable_closure(self):
def f():
s = set((1, 2)) # mutable set is unhashable
@@ -179,6 +203,51 @@ class CloudPickleTest(unittest.TestCase):
g = pickle_depickle(f())
self.assertEqual(g(), 2)
+ def test_dynamically_generated_class_that_uses_super(self):
+
+ class Base(object):
+ def method(self):
+ return 1
+
+ class Derived(Base):
+ "Derived Docstring"
+ def method(self):
+ return super(Derived, self).method() + 1
+
+ self.assertEqual(Derived().method(), 2)
+
+ # Pickle and unpickle the class.
+ UnpickledDerived = pickle_depickle(Derived)
+ self.assertEqual(UnpickledDerived().method(), 2)
+
+ # We have special logic for handling __doc__ because it's a readonly
+ # attribute on PyPy.
+ self.assertEqual(UnpickledDerived.__doc__, "Derived Docstring")
+
+ # Pickle and unpickle an instance.
+ orig_d = Derived()
+ d = pickle_depickle(orig_d)
+ self.assertEqual(d.method(), 2)
+
+ def test_cycle_in_classdict_globals(self):
+
+ class C(object):
+
+ def it_works(self):
+ return "woohoo!"
+
+ C.C_again = C
+ C.instance_of_C = C()
+
+ depickled_C = pickle_depickle(C)
+ depickled_instance = pickle_depickle(C())
+
+ # Test instance of depickled class.
+ self.assertEqual(depickled_C().it_works(), "woohoo!")
+ self.assertEqual(depickled_C.C_again().it_works(), "woohoo!")
+ self.assertEqual(depickled_C.instance_of_C.it_works(), "woohoo!")
+ self.assertEqual(depickled_instance.it_works(), "woohoo!")
+
@pytest.mark.skipif(sys.version_info >= (3, 4)
and sys.version_info < (3, 4, 3),
reason="subprocess has a bug in 3.4.0 to 3.4.2")
@@ -507,6 +576,110 @@ class CloudPickleTest(unittest.TestCase):
msg='cell contents not set correctly',
)
+ def test_logger(self):
+ logger = logging.getLogger('cloudpickle.dummy_test_logger')
+ pickled = pickle_depickle(logger)
+ self.assertTrue(pickled is logger, (pickled, logger))
+
+ dumped = cloudpickle.dumps(logger)
+
+ code = """if 1:
+ import cloudpickle, logging
+
+ logging.basicConfig(level=logging.INFO)
+ logger = cloudpickle.loads(%(dumped)r)
+ logger.info('hello')
+ """ % locals()
+ proc = subprocess.Popen([sys.executable, "-c", code],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT)
+ out, _ = proc.communicate()
+ self.assertEqual(proc.wait(), 0)
+ self.assertEqual(out.strip().decode(),
+ 'INFO:cloudpickle.dummy_test_logger:hello')
+
+ def test_abc(self):
+
+ @abc.abstractmethod
+ def foo(self):
+ raise NotImplementedError('foo')
+
+ # Invoke the metaclass directly rather than using class syntax for
+ # python 2/3 compat.
+ AbstractClass = abc.ABCMeta('AbstractClass', (object,), {'foo': foo})
+
+ class ConcreteClass(AbstractClass):
+ def foo(self):
+ return 'it works!'
+
+ depickled_base = pickle_depickle(AbstractClass)
+ depickled_class = pickle_depickle(ConcreteClass)
+ depickled_instance = pickle_depickle(ConcreteClass())
+
+ self.assertEqual(depickled_class().foo(), 'it works!')
+ self.assertEqual(depickled_instance.foo(), 'it works!')
+
+ # assertRaises doesn't return a contextmanager in python 2.6 :(.
+ self.failUnlessRaises(TypeError, depickled_base)
+
+ class DepickledBaseSubclass(depickled_base):
+ def foo(self):
+ return 'it works for realz!'
+
+ self.assertEqual(DepickledBaseSubclass().foo(), 'it works for realz!')
+
+ @pytest.mark.skipif(not HAVE_WEAKSET, reason="WeakSet doesn't exist")
+ def test_weakset_identity_preservation(self):
+ # Test that weaksets don't lose all their inhabitants if they're
+ # pickled in a larger data structure that includes other references to
+ # their inhabitants.
+
+ class SomeClass(object):
+ def __init__(self, x):
+ self.x = x
+
+ obj1, obj2, obj3 = SomeClass(1), SomeClass(2), SomeClass(3)
+
+ things = [weakref.WeakSet([obj1, obj2]), obj1, obj2, obj3]
+ result = pickle_depickle(things)
+
+ weakset, depickled1, depickled2, depickled3 = result
+
+ self.assertEqual(depickled1.x, 1)
+ self.assertEqual(depickled2.x, 2)
+ self.assertEqual(depickled3.x, 3)
+ self.assertEqual(len(weakset), 2)
+
+ self.assertEqual(set(weakset), set([depickled1, depickled2]))
+
+ def test_ignoring_whichmodule_exception(self):
+ class FakeModule(object):
+ def __getattr__(self, name):
+ # This throws an exception while looking up within
+ # pickle.whichimodule.
+ raise Exception()
+
+ class Foo(object):
+ __module__ = None
+
+ def foo(self):
+ return "it works!"
+
+ def foo():
+ return "it works!"
+
+ foo.__module__ = None
+
+ sys.modules["_fake_module"] = FakeModule()
+ try:
+ # Test whichmodule in save_global.
+ self.assertEqual(pickle_depickle(Foo()).foo(), "it works!")
+
+ # Test whichmodule in save_function.
+ self.assertEqual(pickle_depickle(foo)(), "it works!")
+ finally:
+ sys.modules.pop("_fake_module", None)
+
if __name__ == '__main__':
unittest.main()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/cloudpickle.git
More information about the Python-modules-commits
mailing list