[Python-modules-commits] [cloudpickle] 02/09: New upstream version 0.3.1
Diane Trout
diane at moszumanska.debian.org
Fri Oct 13 17:25:39 UTC 2017
This is an automated email from the git hooks/post-receive script.
diane pushed a commit to branch master
in repository cloudpickle.
commit bd15b7a3a1e1446160b3a637ad6250809685a6a6
Author: Diane Trout <diane at ghic.org>
Date: Thu Jun 1 20:52:24 2017 -0700
New upstream version 0.3.1
---
PKG-INFO | 2 +-
cloudpickle.egg-info/PKG-INFO | 2 +-
cloudpickle/__init__.py | 2 +-
cloudpickle/cloudpickle.py | 177 ++++++++++++++++++++++++++++++++++++-----
setup.cfg | 1 -
setup.py | 2 +-
tests/cloudpickle_file_test.py | 16 ++--
tests/cloudpickle_test.py | 151 ++++++++++++++++++++++++++++++++++-
8 files changed, 316 insertions(+), 37 deletions(-)
diff --git a/PKG-INFO b/PKG-INFO
index 412a7c0..cf7660e 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.2.2
+Version: 0.3.1
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
diff --git a/cloudpickle.egg-info/PKG-INFO b/cloudpickle.egg-info/PKG-INFO
index 412a7c0..cf7660e 100644
--- a/cloudpickle.egg-info/PKG-INFO
+++ b/cloudpickle.egg-info/PKG-INFO
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.2.2
+Version: 0.3.1
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
diff --git a/cloudpickle/__init__.py b/cloudpickle/__init__.py
index 46b2f1a..245645e 100644
--- a/cloudpickle/__init__.py
+++ b/cloudpickle/__init__.py
@@ -2,4 +2,4 @@ from __future__ import absolute_import
from cloudpickle.cloudpickle import *
-__version__ = '0.2.2'
+__version__ = '0.3.1'
diff --git a/cloudpickle/cloudpickle.py b/cloudpickle/cloudpickle.py
index e8f4223..030d44a 100644
--- a/cloudpickle/cloudpickle.py
+++ b/cloudpickle/cloudpickle.py
@@ -56,6 +56,7 @@ import traceback
import types
import weakref
+
if sys.version < '3':
from pickle import Pickler
try:
@@ -69,6 +70,92 @@ else:
from io import BytesIO as StringIO
PY3 = True
+
+def _make_cell_set_template_code():
+ """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
+
+ Notes
+ -----
+ In Python 3, we could use an easier function:
+
+ .. code-block:: python
+
+ def f():
+ cell = None
+
+ def _stub(value):
+ nonlocal cell
+ cell = value
+
+ return _stub
+
+ _cell_set_template_code = f()
+
+ This function is _only_ a LOAD_FAST(arg); STORE_DEREF, but that is
+ invalid syntax on Python 2. If we use this function we also don't need
+ to do the weird freevars/cellvars swap below
+ """
+ def inner(value):
+ lambda: cell # make ``cell`` a closure so that we get a STORE_DEREF
+ cell = value
+
+ co = inner.__code__
+
+ # NOTE: we are marking the cell variable as a free variable intentionally
+ # so that we simulate an inner function instead of the outer function. This
+ # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
+ if not PY3:
+ return types.CodeType(
+ co.co_argcount,
+ co.co_nlocals,
+ co.co_stacksize,
+ co.co_flags,
+ co.co_code,
+ co.co_consts,
+ co.co_names,
+ co.co_varnames,
+ co.co_filename,
+ co.co_name,
+ co.co_firstlineno,
+ co.co_lnotab,
+ co.co_cellvars, # this is the trickery
+ (),
+ )
+ else:
+ return types.CodeType(
+ co.co_argcount,
+ co.co_kwonlyargcount,
+ co.co_nlocals,
+ co.co_stacksize,
+ co.co_flags,
+ co.co_code,
+ co.co_consts,
+ co.co_names,
+ co.co_varnames,
+ co.co_filename,
+ co.co_name,
+ co.co_firstlineno,
+ co.co_lnotab,
+ co.co_cellvars, # this is the trickery
+ (),
+ )
+
+
+_cell_set_template_code = _make_cell_set_template_code()
+
+
+def cell_set(cell, value):
+ """Set the value of a closure cell.
+ """
+ return types.FunctionType(
+ _cell_set_template_code,
+ {},
+ '_cell_set_inner',
+ (),
+ (cell,),
+ )(value)
+
+
#relevant opcodes
STORE_GLOBAL = opcode.opmap['STORE_GLOBAL']
DELETE_GLOBAL = opcode.opmap['DELETE_GLOBAL']
@@ -176,11 +263,14 @@ class CloudPickler(Pickler):
"""
mod_name = obj.__name__
# If module is successfully found then it is not a dynamically created module
- try:
- _find_module(mod_name)
+ if hasattr(obj, '__file__'):
is_dynamic = False
- except ImportError:
- is_dynamic = True
+ else:
+ try:
+ _find_module(mod_name)
+ is_dynamic = False
+ except ImportError:
+ is_dynamic = True
self.modules.add(obj)
if is_dynamic:
@@ -238,7 +328,7 @@ class CloudPickler(Pickler):
# a builtin_function_or_method which comes in as an attribute of some
# object (e.g., object.__new__, itertools.chain.from_iterable) will end
# up with modname "__main__" and so end up here. But these functions
- # have no __code__ attribute in CPython, so the handling for
+ # have no __code__ attribute in CPython, so the handling for
# user-defined functions below will fail.
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
@@ -282,6 +372,26 @@ class CloudPickler(Pickler):
self.memoize(obj)
dispatch[types.FunctionType] = save_function
+ def _save_subimports(self, code, top_level_dependencies):
+ """
+ Ensure de-pickler imports any package child-modules that
+ are needed by the function
+ """
+ # check if any known dependency is an imported package
+ for x in top_level_dependencies:
+ if isinstance(x, types.ModuleType) and x.__package__:
+ # check if the package has any currently loaded sub-imports
+ prefix = x.__name__ + '.'
+ for name, module in sys.modules.items():
+ if name.startswith(prefix):
+ # check whether the function can address the sub-module
+ tokens = set(name[len(prefix):].split('.'))
+ if not tokens - set(code.co_names):
+ # ensure unpickler executes this import
+ self.save(module)
+ # then discards the reference to it
+ self.write(pickle.POP)
+
def save_function_tuple(self, func):
""" Pickles an actual func object.
@@ -302,14 +412,23 @@ class CloudPickler(Pickler):
save = self.save
write = self.write
- code, f_globals, defaults, closure, dct, base_globals = self.extract_func_data(func)
+ code, f_globals, defaults, closure_values, dct, base_globals = self.extract_func_data(func)
save(_fill_function) # skeleton function updater
write(pickle.MARK) # beginning of tuple that _fill_function expects
+ self._save_subimports(
+ code,
+ itertools.chain(f_globals.values(), closure_values or ()),
+ )
+
# create a skeleton function object and memoize it
save(_make_skel_func)
- save((code, closure, base_globals))
+ save((
+ code,
+ len(closure_values) if closure_values is not None else -1,
+ base_globals,
+ ))
write(pickle.REDUCE)
self.memoize(func)
@@ -317,6 +436,7 @@ class CloudPickler(Pickler):
save(f_globals)
save(defaults)
save(dct)
+ save(closure_values)
write(pickle.TUPLE)
write(pickle.REDUCE) # applies _fill_function on the tuple
@@ -354,7 +474,7 @@ class CloudPickler(Pickler):
def extract_func_data(self, func):
"""
Turn the function into a tuple of data necessary to recreate it:
- code, globals, defaults, closure, dict
+ code, globals, defaults, closure_values, dict
"""
code = func.__code__
@@ -371,7 +491,10 @@ class CloudPickler(Pickler):
defaults = func.__defaults__
# process closure
- closure = [c.cell_contents for c in func.__closure__] if func.__closure__ else []
+ closure = (
+ [c.cell_contents for c in func.__closure__]
+ if func.__closure__ is not None else None
+ )
# save the dict
dct = func.__dict__
@@ -773,38 +896,46 @@ def _gen_ellipsis():
def _gen_not_implemented():
return NotImplemented
-def _fill_function(func, globals, defaults, dict):
+def _fill_function(func, globals, defaults, dict, closure_values):
""" Fills in the rest of function data into the skeleton function object
that were created via _make_skel_func().
- """
+ """
func.__globals__.update(globals)
func.__defaults__ = defaults
func.__dict__ = dict
- return func
+ cells = func.__closure__
+ if cells is not None:
+ for cell, value in zip(cells, closure_values):
+ cell_set(cell, value)
+ return func
-def _make_cell(value):
- return (lambda: value).__closure__[0]
+def _make_empty_cell():
+ if False:
+ # trick the compiler into creating an empty cell in our lambda
+ cell = None
+ raise AssertionError('this route should not be executed')
-def _reconstruct_closure(values):
- return tuple([_make_cell(v) for v in values])
+ return (lambda: cell).__closure__[0]
-def _make_skel_func(code, closures, base_globals = None):
+def _make_skel_func(code, cell_count, base_globals=None):
""" Creates a skeleton function object that contains just the provided
code and the correct number of cells in func_closure. All other
func attributes (e.g. func_globals) are empty.
"""
- closure = _reconstruct_closure(closures) if closures else None
-
if base_globals is None:
base_globals = {}
base_globals['__builtins__'] = __builtins__
- return types.FunctionType(code, base_globals,
- None, None, closure)
+ closure = (
+ tuple(_make_empty_cell() for _ in range(cell_count))
+ if cell_count >= 0 else
+ None
+ )
+ return types.FunctionType(code, base_globals, None, None, closure)
def _find_module(mod_name):
@@ -817,7 +948,9 @@ def _find_module(mod_name):
if path is not None:
path = [path]
file, path, description = imp.find_module(part, path)
- return file, path, description
+ if file is not None:
+ file.close()
+ return path, description
"""Constructors for 3rd party libraries
Note: These can never be renamed due to client compatibility issues"""
diff --git a/setup.cfg b/setup.cfg
index 6c71b61..1e3eb36 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -4,5 +4,4 @@ universal = 1
[egg_info]
tag_build =
tag_date = 0
-tag_svn_revision = 0
diff --git a/setup.py b/setup.py
index d32149e..e59d90b 100644
--- a/setup.py
+++ b/setup.py
@@ -8,7 +8,7 @@ except ImportError:
dist = setup(
name='cloudpickle',
- version='0.2.2',
+ version='0.3.1',
description='Extended pickling support for Python objects',
author='Cloudpipe',
author_email='cloudpipe at googlegroups.com',
diff --git a/tests/cloudpickle_file_test.py b/tests/cloudpickle_file_test.py
index f98cf07..4799359 100644
--- a/tests/cloudpickle_file_test.py
+++ b/tests/cloudpickle_file_test.py
@@ -32,7 +32,7 @@ class CloudPickleFileTests(unittest.TestCase):
# Empty file
open(self.tmpfilepath, 'w').close()
with open(self.tmpfilepath, 'r') as f:
- self.assertEquals('', pickle.loads(cloudpickle.dumps(f)).read())
+ self.assertEqual('', pickle.loads(cloudpickle.dumps(f)).read())
os.remove(self.tmpfilepath)
def test_closed_file(self):
@@ -51,7 +51,7 @@ class CloudPickleFileTests(unittest.TestCase):
# Open for reading
with open(self.tmpfilepath, 'r') as f:
new_f = pickle.loads(cloudpickle.dumps(f))
- self.assertEquals(self.teststring, new_f.read())
+ self.assertEqual(self.teststring, new_f.read())
os.remove(self.tmpfilepath)
def test_w_mode(self):
@@ -68,7 +68,7 @@ class CloudPickleFileTests(unittest.TestCase):
f.write(self.teststring)
f.seek(0)
new_f = pickle.loads(cloudpickle.dumps(f))
- self.assertEquals(self.teststring, new_f.read())
+ self.assertEqual(self.teststring, new_f.read())
os.remove(self.tmpfilepath)
def test_seek(self):
@@ -78,11 +78,11 @@ class CloudPickleFileTests(unittest.TestCase):
f.seek(4)
unpickled = pickle.loads(cloudpickle.dumps(f))
# unpickled StringIO is at position 4
- self.assertEquals(4, unpickled.tell())
- self.assertEquals(self.teststring[4:], unpickled.read())
+ self.assertEqual(4, unpickled.tell())
+ self.assertEqual(self.teststring[4:], unpickled.read())
# but unpickled StringIO also contained the start
unpickled.seek(0)
- self.assertEquals(self.teststring, unpickled.read())
+ self.assertEqual(self.teststring, unpickled.read())
os.remove(self.tmpfilepath)
@pytest.mark.skipif(sys.version_info >= (3,),
@@ -94,12 +94,12 @@ class CloudPickleFileTests(unittest.TestCase):
f = fp.file
# FIXME this doesn't work yet: cloudpickle.dumps(fp)
newfile = pickle.loads(cloudpickle.dumps(f))
- self.assertEquals(self.teststring, newfile.read())
+ self.assertEqual(self.teststring, newfile.read())
def test_pickling_special_file_handles(self):
# Warning: if you want to run your tests with nose, add -s option
for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415
- self.assertEquals(out, pickle.loads(cloudpickle.dumps(out)))
+ self.assertEqual(out, pickle.loads(cloudpickle.dumps(out)))
self.assertRaises(pickle.PicklingError,
lambda: cloudpickle.dumps(sys.stdin))
diff --git a/tests/cloudpickle_test.py b/tests/cloudpickle_test.py
index a1dec11..19f1faf 100644
--- a/tests/cloudpickle_test.py
+++ b/tests/cloudpickle_test.py
@@ -9,6 +9,8 @@ import functools
import itertools
import platform
import textwrap
+import base64
+import subprocess
try:
# try importing numpy and scipy. These are not hard dependencies and
@@ -36,7 +38,7 @@ except ImportError:
from io import BytesIO
import cloudpickle
-from cloudpickle.cloudpickle import _find_module
+from cloudpickle.cloudpickle import _find_module, _make_empty_cell, cell_set
from .testutils import subprocess_pickle_echo
@@ -91,7 +93,7 @@ class CloudPickleTest(unittest.TestCase):
def test_pickling_file_handles(self):
out1 = sys.stderr
out2 = pickle.loads(cloudpickle.dumps(out1))
- self.assertEquals(out1, out2)
+ self.assertEqual(out1, out2)
def test_func_globals(self):
class Unpicklable(object):
@@ -131,6 +133,52 @@ class CloudPickleTest(unittest.TestCase):
f2 = lambda x: f1(x) // b
self.assertEqual(pickle_depickle(f2)(1), 1)
+ def test_recursive_closure(self):
+ def f1():
+ def g():
+ return g
+ return g
+
+ def f2(base):
+ def g(n):
+ return base if n <= 1 else n * g(n - 1)
+ return g
+
+ g1 = pickle_depickle(f1())
+ self.assertEqual(g1(), g1)
+
+ g2 = pickle_depickle(f2(2))
+ self.assertEqual(g2(5), 240)
+
+ def test_closure_none_is_preserved(self):
+ def f():
+ """a function with no closure cells
+ """
+
+ self.assertTrue(
+ f.__closure__ is None,
+ msg='f actually has closure cells!',
+ )
+
+ g = pickle_depickle(f)
+
+ self.assertTrue(
+ g.__closure__ is None,
+ msg='g now has closure cells even though f does not',
+ )
+
+ def test_unhashable_closure(self):
+ def f():
+ s = set((1, 2)) # mutable set is unhashable
+
+ def g():
+ return len(s)
+
+ return g
+
+ g = pickle_depickle(f())
+ self.assertEqual(g(), 2)
+
@pytest.mark.skipif(sys.version_info >= (3, 4)
and sys.version_info < (3, 4, 3),
reason="subprocess has a bug in 3.4.0 to 3.4.2")
@@ -360,6 +408,105 @@ class CloudPickleTest(unittest.TestCase):
self.assertTrue(f2 is f3)
self.assertEqual(f2(), res)
+ def test_submodule(self):
+ # Function that refers (by attribute) to a sub-module of a package.
+
+ # Choose any module NOT imported by __init__ of its parent package
+ # examples in standard library include:
+ # - http.cookies, unittest.mock, curses.textpad, xml.etree.ElementTree
+
+ global xml # imitate performing this import at top of file
+ import xml.etree.ElementTree
+ def example():
+ x = xml.etree.ElementTree.Comment # potential AttributeError
+
+ s = cloudpickle.dumps(example)
+
+ # refresh the environment, i.e., unimport the dependency
+ del xml
+ for item in list(sys.modules):
+ if item.split('.')[0] == 'xml':
+ del sys.modules[item]
+
+ # deserialise
+ f = pickle.loads(s)
+ f() # perform test for error
+
+ def test_submodule_closure(self):
+ # Same as test_submodule except the package is not a global
+ def scope():
+ import xml.etree.ElementTree
+ def example():
+ x = xml.etree.ElementTree.Comment # potential AttributeError
+ return example
+ example = scope()
+
+ s = cloudpickle.dumps(example)
+
+ # refresh the environment (unimport dependency)
+ for item in list(sys.modules):
+ if item.split('.')[0] == 'xml':
+ del sys.modules[item]
+
+ f = cloudpickle.loads(s)
+ f() # test
+
+ def test_multiprocess(self):
+ # running a function pickled by another process (a la dask.distributed)
+ def scope():
+ import curses.textpad
+ def example():
+ x = xml.etree.ElementTree.Comment
+ x = curses.textpad.Textbox
+ return example
+ global xml
+ import xml.etree.ElementTree
+ example = scope()
+
+ s = cloudpickle.dumps(example)
+
+ # choose "subprocess" rather than "multiprocessing" because the latter
+ # library uses fork to preserve the parent environment.
+ command = ("import pickle, base64; "
+ "pickle.loads(base64.b32decode('" +
+ base64.b32encode(s).decode('ascii') +
+ "'))()")
+ assert not subprocess.call([sys.executable, '-c', command])
+
+ def test_import(self):
+ # like test_multiprocess except subpackage modules referenced directly
+ # (unlike test_submodule)
+ global etree
+ def scope():
+ import curses.textpad as foobar
+ def example():
+ x = etree.Comment
+ x = foobar.Textbox
+ return example
+ example = scope()
+ import xml.etree.ElementTree as etree
+
+ s = cloudpickle.dumps(example)
+
+ command = ("import pickle, base64; "
+ "pickle.loads(base64.b32decode('" +
+ base64.b32encode(s).decode('ascii') +
+ "'))()")
+ assert not subprocess.call([sys.executable, '-c', command])
+
+ def test_cell_manipulation(self):
+ cell = _make_empty_cell()
+
+ with pytest.raises(ValueError):
+ cell.cell_contents
+
+ ob = object()
+ cell_set(cell, ob)
+ self.assertTrue(
+ cell.cell_contents is ob,
+ msg='cell contents not set correctly',
+ )
+
if __name__ == '__main__':
unittest.main()
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/python-modules/packages/cloudpickle.git
More information about the Python-modules-commits
mailing list